JUnit version 4.13.2
.21:04:09.390 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783544649383
21:04:09.445 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
21:04:09.477 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:04:09.490 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQ9WW30/test-tmp
21:04:09.515 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
21:04:09.517 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQ9WW30/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
21:04:09.685 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:04:09.696 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:04:09.697 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.698920 31683 mini_oidc.cc:150] Starting JWKS server
21:04:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.700722 31683 webserver.cc:534] Webserver started at http://127.0.0.1:33147/ using document root <none> and password file <none>
21:04:09.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.700888 31683 mini_oidc.cc:186] Starting OIDC Discovery server
21:04:09.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.701138 31683 webserver.cc:534] Webserver started at http://127.0.0.1:44627/ using document root <none> and password file <none>
21:04:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:04:09Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:04:09.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:04:09Z Disabled control of system clock
21:04:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.717056 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/wal
21:04:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data
21:04:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/logs
21:04:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data/info.pb
21:04:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.254:45497
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.254
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33147/jwks.json with env {}
21:04:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.804850 31694 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.805022 31694 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.805071 31694 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.807837 31694 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:04:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.807878 31694 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.807894 31694 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.807909 31694 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:04:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.807924 31694 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.807935 31694 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33147/jwks.json
21:04:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.811435 31694 master_runner.cc:387] Master server non-default flags:
21:04:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data
21:04:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/wal
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.254:45497
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33147/jwks.json
21:04:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data/info.pb
21:04:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.254
21:04:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:09.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31694
21:04:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/logs
21:04:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.812180 31694 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.813150 31694 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.818074 31699 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.818065 31700 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.818065 31702 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.820482 31694 server_base.cc:1061] running on GCE node
21:04:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.820943 31694 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.821712 31694 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.822860 31694 hybrid_clock.cc:648] HybridClock initialized: now 1783544649822812 us; error 61 us; skew 500 ppm
21:04:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.824368 31694 webserver.cc:534] Webserver started at http://127.30.240.254:44647/ using document root <none> and password file <none>
21:04:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.824777 31694 fs_manager.cc:362] Metadata directory not provided
21:04:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.824834 31694 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.825006 31694 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.826233 31694 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data/instance:
21:04:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "769c06c8e80941e88814cad75571c3c8"
21:04:09.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:09 on dist-test-slave-dr23"
21:04:09.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.826586 31694 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/wal/instance:
21:04:09.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "769c06c8e80941e88814cad75571c3c8"
21:04:09.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:09 on dist-test-slave-dr23"
21:04:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.829403 31694 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.001s	sys 0.003s
21:04:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.831175 31708 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.832036 31694 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.832159 31694 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/wal
21:04:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "769c06c8e80941e88814cad75571c3c8"
21:04:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:09 on dist-test-slave-dr23"
21:04:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.832247 31694 fs_report.cc:389] FS layout report
21:04:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/wal
21:04:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/wal
21:04:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data/data
21:04:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:09.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:09.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:09.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.857050 31694 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:09.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.857564 31694 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:09.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.857721 31694 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.863559 31760 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.254:45497 every 8 connection(s)
21:04:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.863559 31694 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.254:45497
21:04:09.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.864461 31694 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/data/info.pb
21:04:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.866410 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 31694
21:04:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.866536 31683 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-0/wal/instance
21:04:09.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.867142 31761 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.867931 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/wal
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/logs
21:04:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data/info.pb
21:04:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.253:42945
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.253
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33147/jwks.json with env {}
21:04:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.870704 31761 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.881260 31761 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } has no permanent_uuid. Determining permanent_uuid...
21:04:09.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.882009 31709 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.253:42945: connect: Connection refused (error 111)
21:04:09.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.882824 31761 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.253:42945: Network error: Client connection negotiation failed: client connection to 127.30.240.253:42945: connect: Connection refused (error 111)
21:04:09.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.932010 31761 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } attempt: 1
21:04:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.933280 31761 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.253:42945: Network error: Client connection negotiation failed: client connection to 127.30.240.253:42945: connect: Connection refused (error 111)
21:04:09.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.962226 31763 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:09.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.962373 31763 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:09.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.962412 31763 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.965437 31763 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:04:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.965482 31763 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.965499 31763 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.965514 31763 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:04:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.965529 31763 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.965541 31763 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33147/jwks.json
21:04:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.969981 31763 master_runner.cc:387] Master server non-default flags:
21:04:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data
21:04:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/wal
21:04:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.253:42945
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33147/jwks.json
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data/info.pb
21:04:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.253
21:04:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31763
21:04:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/logs
21:04:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.970695 31763 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.971797 31763 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.976814 31771 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.976941 31774 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:09.977042 31772 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.977548 31763 server_base.cc:1061] running on GCE node
21:04:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.977937 31763 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.978767 31763 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.979947 31763 hybrid_clock.cc:648] HybridClock initialized: now 1783544649979899 us; error 68 us; skew 500 ppm
21:04:09.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.981328 31763 webserver.cc:534] Webserver started at http://127.30.240.253:42087/ using document root <none> and password file <none>
21:04:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.981832 31763 fs_manager.cc:362] Metadata directory not provided
21:04:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.981905 31763 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.982088 31763 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:09.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.983487 31763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data/instance:
21:04:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1d3b4db84884a5b8806fe019b064e4a"
21:04:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:09 on dist-test-slave-dr23"
21:04:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.983996 31763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/wal/instance:
21:04:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1d3b4db84884a5b8806fe019b064e4a"
21:04:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:09 on dist-test-slave-dr23"
21:04:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.986883 31763 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.001s	sys 0.001s
21:04:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.989007 31780 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.990029 31763 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.990120 31763 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/wal
21:04:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1d3b4db84884a5b8806fe019b064e4a"
21:04:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:09 on dist-test-slave-dr23"
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:09.990190 31763 fs_report.cc:389] FS layout report
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/wal
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/wal
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data/data
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.001433 31761 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } attempt: 2
21:04:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.002471 31761 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.253:42945: Network error: Client connection negotiation failed: client connection to 127.30.240.253:42945: connect: Connection refused (error 111)
21:04:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.022436 31763 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.022912 31763 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.023026 31763 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.029088 31763 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.253:42945
21:04:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.029098 31832 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.253:42945 every 8 connection(s)
21:04:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.030079 31763 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/data/info.pb
21:04:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.032392 31833 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.034017 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 31763
21:04:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.034104 31683 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-1/wal/instance
21:04:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.034690 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/wal
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/logs
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data/info.pb
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.252:45579
21:04:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.252
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:10.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33147/jwks.json with env {}
21:04:10.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.035172 31833 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } has no permanent_uuid. Determining permanent_uuid...
21:04:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.044480 31833 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } has no permanent_uuid. Determining permanent_uuid...
21:04:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.049752 31833 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } has no permanent_uuid. Determining permanent_uuid...
21:04:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.050412 31781 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.252:45579: connect: Connection refused (error 111)
21:04:10.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.051198 31833 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.252:45579: Network error: Client connection negotiation failed: client connection to 127.30.240.252:45579: connect: Connection refused (error 111)
21:04:10.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.093645 31761 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } attempt: 3
21:04:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.097539 31761 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } has no permanent_uuid. Determining permanent_uuid...
21:04:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.098799 31761 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.252:45579: Network error: Client connection negotiation failed: client connection to 127.30.240.252:45579: connect: Connection refused (error 111)
21:04:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.100355 31833 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } attempt: 1
21:04:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.101555 31833 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.252:45579: Network error: Client connection negotiation failed: client connection to 127.30.240.252:45579: connect: Connection refused (error 111)
21:04:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.126655 31835 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.126801 31835 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.126837 31835 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.129690 31835 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:04:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.129735 31835 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.129755 31835 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.129771 31835 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:04:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.129788 31835 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.129803 31835 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33147/jwks.json
21:04:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.129992 31761 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } attempt: 1
21:04:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.131022 31761 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.252:45579: Network error: Client connection negotiation failed: client connection to 127.30.240.252:45579: connect: Connection refused (error 111)
21:04:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.133493 31835 master_runner.cc:387] Master server non-default flags:
21:04:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/wal
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.252:45579
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33147/jwks.json
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data/info.pb
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.252
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31835
21:04:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/logs
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.134238 31835 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.135020 31835 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.140266 31843 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.140328 31844 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.140283 31846 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.140651 31835 server_base.cc:1061] running on GCE node
21:04:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.141088 31835 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.141903 31835 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.143066 31835 hybrid_clock.cc:648] HybridClock initialized: now 1783544650143022 us; error 57 us; skew 500 ppm
21:04:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.144665 31835 webserver.cc:534] Webserver started at http://127.30.240.252:38817/ using document root <none> and password file <none>
21:04:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.145170 31835 fs_manager.cc:362] Metadata directory not provided
21:04:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.145231 31835 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.145381 31835 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.147042 31835 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data/instance:
21:04:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "383a192ea61f49f48c1ad2b62dc3b8f1"
21:04:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.147526 31835 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/wal/instance:
21:04:10.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "383a192ea61f49f48c1ad2b62dc3b8f1"
21:04:10.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.151125 31835 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.002s	sys 0.004s
21:04:10.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.153182 31852 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.154132 31835 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.154210 31835 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/wal
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "383a192ea61f49f48c1ad2b62dc3b8f1"
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.154291 31835 fs_report.cc:389] FS layout report
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/wal
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/wal
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data/data
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:10.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.169703 31833 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } attempt: 2
21:04:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.170780 31833 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.240.252:45579: Network error: Client connection negotiation failed: client connection to 127.30.240.252:45579: connect: Connection refused (error 111)
21:04:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.182554 31835 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.183041 31835 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:10.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.183153 31835 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.188796 31835 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.252:45579
21:04:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.188819 31904 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.252:45579 every 8 connection(s)
21:04:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.189550 31835 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/data/info.pb
21:04:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.191679 31905 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:10.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.193989 31905 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } has no permanent_uuid. Determining permanent_uuid...
21:04:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.199602 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 31835
21:04:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.199699 31683 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/master-2/wal/instance
21:04:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.201759 31905 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } has no permanent_uuid. Determining permanent_uuid...
21:04:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.205286 31905 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } has no permanent_uuid. Determining permanent_uuid...
21:04:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.206167 31761 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } attempt: 2
21:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.213407 31761 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8: Bootstrap starting.
21:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.213485 31905 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: Bootstrap starting.
21:04:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.215400 31905 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: Neither blocks nor log segments found. Creating new log.
21:04:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.215466 31761 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8: Neither blocks nor log segments found. Creating new log.
21:04:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.216077 31761 log.cc:826] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8: Log is configured to *not* fsync() on all Append() calls
21:04:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.216506 31905 log.cc:826] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: Log is configured to *not* fsync() on all Append() calls
21:04:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.217473 31761 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8: No bootstrap required, opened a new log
21:04:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.217661 31905 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: No bootstrap required, opened a new log
21:04:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.219717 31905 raft_consensus.cc:359] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.219916 31905 raft_consensus.cc:385] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.219969 31905 raft_consensus.cc:740] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 383a192ea61f49f48c1ad2b62dc3b8f1, State: Initialized, Role: FOLLOWER
21:04:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.220407 31905 consensus_queue.cc:260] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.220489 31761 raft_consensus.cc:359] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.220630 31761 raft_consensus.cc:385] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.220685 31761 raft_consensus.cc:740] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 769c06c8e80941e88814cad75571c3c8, State: Initialized, Role: FOLLOWER
21:04:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.221055 31761 consensus_queue.cc:260] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.221371 31905 sys_catalog.cc:565] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [sys.catalog]: configured and running, proceeding with master startup.
21:04:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.223223 31915 sys_catalog.cc:455] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.224287 31915 sys_catalog.cc:458] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [sys.catalog]: This master's current role is: FOLLOWER
21:04:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.224239 31835 master_runner.cc:187] Error getting master registration for 127.30.240.254:45497: OK, instance_id { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" instance_seqno: 1783544649862132 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.226121 31835 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:04:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.226791 31926 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.226864 31926 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.228018 31928 sys_catalog.cc:455] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.228129 31928 sys_catalog.cc:458] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: This master's current role is: FOLLOWER
21:04:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.228600 31761 sys_catalog.cc:565] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: configured and running, proceeding with master startup.
21:04:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.233764 31939 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.233831 31939 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.233088 31694 master_runner.cc:187] Error getting master registration for 127.30.240.253:42945: OK, instance_id { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" instance_seqno: 1783544650027481 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.234649 31694 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:04:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.236181 31928 raft_consensus.cc:493] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.236297 31928 raft_consensus.cc:515] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.237063 31928 leader_election.cc:290] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1d3b4db84884a5b8806fe019b064e4a (127.30.240.253:42945), 383a192ea61f49f48c1ad2b62dc3b8f1 (127.30.240.252:45579)
21:04:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.237473 31879 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "769c06c8e80941e88814cad75571c3c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" is_pre_election: true
21:04:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.237689 31879 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 769c06c8e80941e88814cad75571c3c8 in term 0.
21:04:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.238052 31709 leader_election.cc:304] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 383a192ea61f49f48c1ad2b62dc3b8f1, 769c06c8e80941e88814cad75571c3c8; no voters: 
21:04:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.238287 31928 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.238379 31928 raft_consensus.cc:493] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.238420 31928 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 0 FOLLOWER]: Advancing to term 1
21:04:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.239887 31928 raft_consensus.cc:515] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.240267 31928 leader_election.cc:290] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [CANDIDATE]: Term 1 election: Requested vote from peers e1d3b4db84884a5b8806fe019b064e4a (127.30.240.253:42945), 383a192ea61f49f48c1ad2b62dc3b8f1 (127.30.240.252:45579)
21:04:10.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.240710 31879 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "769c06c8e80941e88814cad75571c3c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1"
21:04:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.240796 31879 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [term 0 FOLLOWER]: Advancing to term 1
21:04:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.241671 31879 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 769c06c8e80941e88814cad75571c3c8 in term 1.
21:04:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.241955 31709 leader_election.cc:304] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 383a192ea61f49f48c1ad2b62dc3b8f1, 769c06c8e80941e88814cad75571c3c8; no voters: 
21:04:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.242161 31928 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 1 FOLLOWER]: Leader election won for term 1
21:04:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.242323 31928 raft_consensus.cc:697] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [term 1 LEADER]: Becoming Leader. State: Replica: 769c06c8e80941e88814cad75571c3c8, State: Running, Role: LEADER
21:04:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.242566 31928 consensus_queue.cc:237] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.244345 31941 sys_catalog.cc:455] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 769c06c8e80941e88814cad75571c3c8. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.244731 31941 sys_catalog.cc:458] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: This master's current role is: LEADER
21:04:10.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.247911 31943 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
21:04:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.249730 31943 catalog_manager.cc:1519] Initializing Kudu cluster ID...
21:04:10.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.257436 31879 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [term 1 FOLLOWER]: Refusing update from remote peer 769c06c8e80941e88814cad75571c3c8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:10.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.258225 31941 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.261603 31915 sys_catalog.cc:455] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 769c06c8e80941e88814cad75571c3c8. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.261687 31915 sys_catalog.cc:458] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [sys.catalog]: This master's current role is: FOLLOWER
21:04:10.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.261912 31833 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } attempt: 3
21:04:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.266371 31941 sys_catalog.cc:455] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.266451 31941 sys_catalog.cc:458] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: This master's current role is: LEADER
21:04:10.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.267848 31944 mvcc.cc:204] Tried to move back new op lower bound from 7305398887447101440 to 7305398887395733504. Current Snapshot: MvccSnapshot[applied={T|T < 7305398887447101440}]
21:04:10.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.268977 31915 sys_catalog.cc:455] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.269068 31915 sys_catalog.cc:458] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1 [sys.catalog]: This master's current role is: FOLLOWER
21:04:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.270218 31945 mvcc.cc:204] Tried to move back new op lower bound from 7305398887447101440 to 7305398887395733504. Current Snapshot: MvccSnapshot[applied={T|T < 7305398887447101440}]
21:04:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.270334 31833 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: Bootstrap starting.
21:04:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.271860 31943 catalog_manager.cc:1382] Generated new cluster ID: 316c97c8f82b416283d39fddf8b84bd6
21:04:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.271905 31943 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
21:04:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.272873 31833 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: Neither blocks nor log segments found. Creating new log.
21:04:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.273731 31833 log.cc:826] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: Log is configured to *not* fsync() on all Append() calls
21:04:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.297273 31833 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: No bootstrap required, opened a new log
21:04:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.300210 31833 raft_consensus.cc:359] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.300967 31833 raft_consensus.cc:385] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.301529 31833 raft_consensus.cc:740] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e1d3b4db84884a5b8806fe019b064e4a, State: Initialized, Role: FOLLOWER
21:04:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.302211 31833 consensus_queue.cc:260] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } }
21:04:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.302318 31943 catalog_manager.cc:1405] Generated new certificate authority record
21:04:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.303185 31943 catalog_manager.cc:1539] Loading token signing keys...
21:04:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.312731 31943 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8: Generated new TSK 0
21:04:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.313637 31943 catalog_manager.cc:1549] Initializing in-progress tserver states...
21:04:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.312454 31833 sys_catalog.cc:565] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [sys.catalog]: configured and running, proceeding with master startup.
21:04:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.312520 31807 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "769c06c8e80941e88814cad75571c3c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e1d3b4db84884a5b8806fe019b064e4a" is_pre_election: true
21:04:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.312700 31805 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [term 0 FOLLOWER]: Advancing to term 1
21:04:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.313390 31806 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "769c06c8e80941e88814cad75571c3c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e1d3b4db84884a5b8806fe019b064e4a"
21:04:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.317046 31805 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [term 1 FOLLOWER]: Refusing update from remote peer 769c06c8e80941e88814cad75571c3c8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.317829 31928 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.326257 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:0
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.326557 31963 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.326637 31963 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [sys.catalog]: This master's current role is: FOLLOWER
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.326706 31963 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 769c06c8e80941e88814cad75571c3c8. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.326752 31963 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [sys.catalog]: This master's current role is: FOLLOWER
21:04:10.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.326807 31963 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.326846 31963 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a [sys.catalog]: This master's current role is: FOLLOWER
21:04:10.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.327097 31964 mvcc.cc:204] Tried to move back new op lower bound from 7305398887669682176 to 7305398887395733504. Current Snapshot: MvccSnapshot[applied={T|T < 7305398887447101440}]
21:04:10.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.328485 31962 catalog_manager.cc:1593] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:10.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.328528 31962 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.455562 31971 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.456216 31971 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.456298 31971 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:10.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.460304 31971 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:10.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.460407 31971 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.193
21:04:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.464627 31971 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:0
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31971
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.465524 31971 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.466534 31971 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.472795 31977 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.472815 31979 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.472954 31976 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.474205 31971 server_base.cc:1061] running on GCE node
21:04:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.474519 31971 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.475025 31971 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.476202 31971 hybrid_clock.cc:648] HybridClock initialized: now 1783544650476154 us; error 58 us; skew 500 ppm
21:04:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.477891 31971 webserver.cc:534] Webserver started at http://127.30.240.193:37005/ using document root <none> and password file <none>
21:04:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.478452 31971 fs_manager.cc:362] Metadata directory not provided
21:04:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.478521 31971 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.478777 31971 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.480540 31971 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/instance:
21:04:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6920c90fa92046599c83c12690baf185"
21:04:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.480998 31971 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal/instance:
21:04:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6920c90fa92046599c83c12690baf185"
21:04:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.484467 31971 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.000s
21:04:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.486675 31985 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.487684 31971 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.487785 31971 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6920c90fa92046599c83c12690baf185"
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.487854 31971 fs_report.cc:389] FS layout report
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/data
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.501246 31971 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.501644 31971 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.501747 31971 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:10.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.502205 31971 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.503050 31971 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.503093 31971 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:04:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.503124 31971 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.503136 31971 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:04:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.512388 31971 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.193:46783
21:04:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.512420 32098 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.193:46783 every 8 connection(s)
21:04:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.513118 31971 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.519300 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 31971
21:04:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.519403 31683 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal/instance
21:04:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.522704 32100 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.522877 32100 heartbeater.cc:461] Registering TS with master...
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.523352 32100 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.523545 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:0
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.525040 32101 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.525110 32101 heartbeater.cc:461] Registering TS with master...
21:04:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.525308 32101 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.526376 31797 ts_manager.cc:194] Registered new tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:10.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.526898 31725 ts_manager.cc:194] Registered new tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:10.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.527824 31725 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.193:44089
21:04:10.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.528188 32099 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:10.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.528256 32099 heartbeater.cc:461] Registering TS with master...
21:04:10.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.528402 32099 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:10.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.529515 31868 ts_manager.cc:194] Registered new tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.609862 32106 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.610040 32106 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.610075 32106 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.612731 32106 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.612808 32106 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.194
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.615875 32106 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:0
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.32106
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.616596 32106 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.617306 32106 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.622215 32112 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.622232 32114 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.622212 32111 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.622831 32106 server_base.cc:1061] running on GCE node
21:04:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.623129 32106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.623481 32106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.624604 32106 hybrid_clock.cc:648] HybridClock initialized: now 1783544650624593 us; error 27 us; skew 500 ppm
21:04:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.625839 32106 webserver.cc:534] Webserver started at http://127.30.240.194:34591/ using document root <none> and password file <none>
21:04:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.626293 32106 fs_manager.cc:362] Metadata directory not provided
21:04:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.626341 32106 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.626467 32106 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.627556 32106 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/instance:
21:04:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.627888 32106 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal/instance:
21:04:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:10.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.630609 32106 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.004s	sys 0.000s
21:04:10.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.632326 32120 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.633081 32106 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.633152 32106 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.633199 32106 fs_report.cc:389] FS layout report
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/data
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.651815 32106 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.652606 32106 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.653067 32106 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.653472 32106 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.654212 32106 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.654251 32106 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:04:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.654281 32106 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.654294 32106 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:04:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.662104 32106 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.194:37435
21:04:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.662175 32233 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.194:37435 every 8 connection(s)
21:04:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.662736 32106 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.670092 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 32106
21:04:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.670188 31683 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal/instance
21:04:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.670372 32236 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.670518 32236 heartbeater.cc:461] Registering TS with master...
21:04:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.670859 32236 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.672108 32234 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.672184 32234 heartbeater.cc:461] Registering TS with master...
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.672356 32234 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.672533 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:0
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.673050 31868 ts_manager.cc:194] Registered new tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.674240 31797 ts_manager.cc:194] Registered new tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.674516 32235 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.674590 32235 heartbeater.cc:461] Registering TS with master...
21:04:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.674770 32235 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.675261 31725 ts_manager.cc:194] Registered new tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.675727 31725 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.194:34527
21:04:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.760722 32241 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.760898 32241 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.760954 32241 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.763726 32241 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.763856 32241 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.195
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.767057 32241 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:0
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.32241
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.767982 32241 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.769129 32241 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.774259 32249 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.774259 32246 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.774705 32241 server_base.cc:1061] running on GCE node
21:04:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:10.774296 32247 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.775138 32241 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.775573 32241 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.776715 32241 hybrid_clock.cc:648] HybridClock initialized: now 1783544650776704 us; error 23 us; skew 500 ppm
21:04:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.777967 32241 webserver.cc:534] Webserver started at http://127.30.240.195:42855/ using document root <none> and password file <none>
21:04:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.778427 32241 fs_manager.cc:362] Metadata directory not provided
21:04:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.778497 32241 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.778673 32241 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:04:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.779978 32241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/instance:
21:04:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.780475 32241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal/instance:
21:04:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.783015 32241 fs_manager.cc:696] Time spent creating directory manager: real 0.002s	user 0.002s	sys 0.000s
21:04:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.784657 32255 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.785365 32241 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.002s	sys 0.000s
21:04:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.785465 32241 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.785534 32241 fs_report.cc:389] FS layout report
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/data
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:10.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.797804 32241 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:10.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.798238 32241 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:10.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.798360 32241 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:10.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.798770 32241 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.799599 32241 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.799649 32241 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:04:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.799687 32241 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:10.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.799705 32241 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:04:10.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.807428 32241 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.195:33321
21:04:10.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.807480 32368 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.195:33321 every 8 connection(s)
21:04:10.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.808260 32241 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.816107 32370 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.816300 32370 heartbeater.cc:461] Registering TS with master...
21:04:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.816376 32371 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.816447 32371 heartbeater.cc:461] Registering TS with master...
21:04:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.816769 32370 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.816769 32371 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.817030 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 32241
21:04:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.817098 31683 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal/instance
21:04:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.817235 32369 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.817288 32369 heartbeater.cc:461] Registering TS with master...
21:04:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.817442 32369 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.817898 31797 ts_manager.cc:194] Registered new tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.817972 31725 ts_manager.cc:194] Registered new tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:10.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.818547 31725 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.195:55451
21:04:10.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.819031 31868 ts_manager.cc:194] Registered new tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.822955 31683 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
21:04:10.823 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:04:10.824 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

21:04:10.832 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.30.240.254"
      port: 45497
    }
    bound_http_address {
      host: "127.30.240.254"
      port: 44647
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.30.240.253"
      port: 42945
    }
    bound_http_address {
      host: "127.30.240.253"
      port: 42087
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.30.240.252"
      port: 45579
    }
    bound_http_address {
      host: "127.30.240.252"
      port: 38817
    }
  }
}

21:04:10.834 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:04:10.836 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.30.240.193"
      port: 46783
    }
    bound_http_address {
      host: "127.30.240.193"
      port: 37005
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.30.240.194"
      port: 37435
    }
    bound_http_address {
      host: "127.30.240.194"
      port: 34591
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.30.240.195"
      port: 33321
    }
    bound_http_address {
      host: "127.30.240.195"
      port: 42855
    }
  }
}

21:04:10.836 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:04:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.865428 31928 sys_catalog.cc:455] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "769c06c8e80941e88814cad75571c3c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "769c06c8e80941e88814cad75571c3c8" member_type: VOTER last_known_addr { host: "127.30.240.254" port: 45497 } } peers { permanent_uuid: "e1d3b4db84884a5b8806fe019b064e4a" member_type: VOTER last_known_addr { host: "127.30.240.253" port: 42945 } } peers { permanent_uuid: "383a192ea61f49f48c1ad2b62dc3b8f1" member_type: VOTER last_known_addr { host: "127.30.240.252" port: 45579 } } }
21:04:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:10.865540 31928 sys_catalog.cc:458] T 00000000000000000000000000000000 P 769c06c8e80941e88814cad75571c3c8 [sys.catalog]: This master's current role is: LEADER
21:04:10.978 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:04:11.025 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
21:04:11.176 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.240.254:45497(127.30.240.254:45497)] connecting to peer
21:04:11.229 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.240.252:45579(127.30.240.252:45579)] connecting to peer
21:04:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.228394 31926 catalog_manager.cc:1294] Loaded cluster ID: 316c97c8f82b416283d39fddf8b84bd6
21:04:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.228461 31926 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: loading cluster ID for follower catalog manager: success
21:04:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.230710 31926 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: acquiring CA information for follower catalog manager: success
21:04:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.231889 31926 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 383a192ea61f49f48c1ad2b62dc3b8f1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:11.232 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.240.253:42945(127.30.240.253:42945)] connecting to peer
21:04:11.245 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.240.252:45579(127.30.240.252:45579)] Successfully connected to peer
21:04:11.245 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.240.254:45497(127.30.240.254:45497)] Successfully connected to peer
21:04:11.246 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.30.240.253:42945(127.30.240.253:42945)] Successfully connected to peer
21:04:11.329 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.240.252
21:04:11.329 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.240.254
21:04:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.329913 31962 catalog_manager.cc:1294] Loaded cluster ID: 316c97c8f82b416283d39fddf8b84bd6
21:04:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.330094 31962 catalog_manager.cc:1587] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: loading cluster ID for follower catalog manager: success
21:04:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.332307 31962 catalog_manager.cc:1609] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: acquiring CA information for follower catalog manager: success
21:04:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.333633 31962 catalog_manager.cc:1637] T 00000000000000000000000000000000 P e1d3b4db84884a5b8806fe019b064e4a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:11.336 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.30.240.253
21:04:11.517 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x9b72409f, L:/127.0.0.1:48722 - R:/127.30.240.252:45579] using SASL/PLAIN
21:04:11.517 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x8c050425, L:/127.0.0.1:33134 - R:/127.30.240.253:42945] using SASL/PLAIN
21:04:11.517 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf2544b0d, L:/127.0.0.1:46532 - R:/127.30.240.254:45497] using SASL/PLAIN
21:04:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.529551 32100 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:11.545 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:11.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=406), Traces: [0ms] refreshing cache from master, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.240.254:45497, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.240.252:45579, [89ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.240.253:42945, [385ms] Sub RPC ConnectToMaster: received response from server master-127.30.240.253:42945: OK, [385ms] Sub RPC ConnectToMaster: received response from server master-127.30.240.254:45497: OK, [385ms] Sub RPC ConnectToMaster: received response from server master-127.30.240.252:45579: OK, deferred=null) after lookup
21:04:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.581465 31725 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46532:
21:04:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126"
21:04:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
21:04:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:04:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:11.582760 31725 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.608458 32168 tablet_service.cc:1511] Processing CreateTablet for tablet 8a5460f709204d5fac0a735d09169599 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126 [id=14b2f1117efc45148622a403e5d43efe]), partition=RANGE (key) PARTITION UNBOUNDED
21:04:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.608457 32033 tablet_service.cc:1511] Processing CreateTablet for tablet 8a5460f709204d5fac0a735d09169599 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126 [id=14b2f1117efc45148622a403e5d43efe]), partition=RANGE (key) PARTITION UNBOUNDED
21:04:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.609378 32033 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8a5460f709204d5fac0a735d09169599. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.609378 32168 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8a5460f709204d5fac0a735d09169599. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.615501 32402 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap starting.
21:04:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.617182 32401 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap starting.
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.618535 32303 tablet_service.cc:1511] Processing CreateTablet for tablet 8a5460f709204d5fac0a735d09169599 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126 [id=14b2f1117efc45148622a403e5d43efe]), partition=RANGE (key) PARTITION UNBOUNDED
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.619176 32401 tablet_bootstrap.cc:654] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Neither blocks nor log segments found. Creating new log.
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.619421 32303 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8a5460f709204d5fac0a735d09169599. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.619946 32401 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Log is configured to *not* fsync() on all Append() calls
21:04:11.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.620976 32402 tablet_bootstrap.cc:654] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Neither blocks nor log segments found. Creating new log.
21:04:11.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.621784 32402 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Log is configured to *not* fsync() on all Append() calls
21:04:11.623 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 14b2f1117efc45148622a403e5d43efe
21:04:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.623579 32402 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: No bootstrap required, opened a new log
21:04:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.623739 32402 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent bootstrapping tablet: real 0.008s	user 0.004s	sys 0.000s
21:04:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.625102 32405 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap starting.
21:04:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.626659 32405 tablet_bootstrap.cc:654] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Neither blocks nor log segments found. Creating new log.
21:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.626848 32402 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.627213 32402 raft_consensus.cc:385] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.627221 32405 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Log is configured to *not* fsync() on all Append() calls
21:04:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.627455 32402 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Initialized, Role: FOLLOWER
21:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.627897 32402 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.628633 32402 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent starting tablet: real 0.005s	user 0.003s	sys 0.000s
21:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.629002 32235 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.629504 32401 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: No bootstrap required, opened a new log
21:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.629698 32401 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent bootstrapping tablet: real 0.013s	user 0.004s	sys 0.000s
21:04:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.630941 32405 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: No bootstrap required, opened a new log
21:04:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.631263 32405 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent bootstrapping tablet: real 0.006s	user 0.003s	sys 0.000s
21:04:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.632936 32401 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.633301 32401 raft_consensus.cc:385] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.633503 32401 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6920c90fa92046599c83c12690baf185, State: Initialized, Role: FOLLOWER
21:04:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.634071 32401 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.634217 32405 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.634497 32405 raft_consensus.cc:385] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.634655 32405 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Initialized, Role: FOLLOWER
21:04:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.634976 32401 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
21:04:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.635195 32405 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.635957 32405 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent starting tablet: real 0.005s	user 0.004s	sys 0.000s
21:04:11.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.636267 32370 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:11.664638 32237 tablet.cc:2406] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:11.768676 32102 tablet.cc:2406] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:11.809764 32372 tablet.cc:2406] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.893173 32407 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.893344 32407 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.894372 32407 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.897325 32409 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.897460 32409 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.898440 32409 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.899324 32053 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185" is_pre_election: true
21:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.899574 32053 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 0.
21:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.899641 32323 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.899786 32323 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 0.
21:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.900059 32122 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185; no voters: 
21:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.900283 32407 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.900352 32407 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.900390 32407 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 0 FOLLOWER]: Advancing to term 1
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.901373 32407 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.901695 32407 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 1 election: Requested vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.902287 32323 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.902376 32323 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 0 FOLLOWER]: Advancing to term 1
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.903324 32323 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 1.
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.903627 32123 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237; no voters: 
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.903738 32053 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185"
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.903828 32053 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 0 FOLLOWER]: Advancing to term 1
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.903913 32407 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 1 FOLLOWER]: Leader election won for term 1
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.904075 32407 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 1 LEADER]: Becoming Leader. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Running, Role: LEADER
21:04:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.904348 32407 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.905115 32053 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 1.
21:04:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.906881 32188 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7" is_pre_election: true
21:04:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.908011 32053 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185" is_pre_election: true
21:04:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.908212 32053 raft_consensus.cc:2393] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 4764cea6067a4b4183944ffb44dfc237 in current term 1: Already voted for candidate 42e2bcf086eb4742a41e54464efe56f7 in this term.
21:04:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.908046 31725 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194). New cstate: current_term: 1 leader_uuid: "42e2bcf086eb4742a41e54464efe56f7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: UNKNOWN } } }
21:04:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.908641 32257 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237; no voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185
21:04:11.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:11.909052 32409 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:04:12.106 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 14b2f1117efc45148622a403e5d43efe
21:04:12.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:12.126 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299995, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:12.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [6ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=null) after lookup
21:04:12.135 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:12.137 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] Successfully connected to peer
21:04:12.197 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x56451624, L:/127.0.0.1:37632 - R:/127.30.240.194:37435] using TOKEN/null
21:04:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.203836 32323 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Refusing update from remote peer 42e2bcf086eb4742a41e54464efe56f7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.203845 32053 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 1 FOLLOWER]: Refusing update from remote peer 42e2bcf086eb4742a41e54464efe56f7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.204427 32407 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.204581 32414 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:12.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.210983 32418 mvcc.cc:204] Tried to move back new op lower bound from 7305398895420731392 to 7305398894202712064. Current Snapshot: MvccSnapshot[applied={T|T < 7305398895420731392}]
21:04:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.212800 32420 mvcc.cc:204] Tried to move back new op lower bound from 7305398895420731392 to 7305398894202712064. Current Snapshot: MvccSnapshot[applied={T|T < 7305398895420731392 or (T in {7305398895420731392})}]
21:04:12.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.214169 32421 mvcc.cc:204] Tried to move back new op lower bound from 7305398895420731392 to 7305398894202712064. Current Snapshot: MvccSnapshot[applied={T|T < 7305398895420731392}]
21:04:12.278 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:12.279 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.194:37435
21:04:12.280 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:12.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:12.281083 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 32106
21:04:12.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:12.293 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] handling channelInactive
21:04:12.293 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state READY due to: connection closed
21:04:12.294 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: connection closed
21:04:12.303 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:12.303 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:12.304 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [4ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [14ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@71392203(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@50e7a10 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@65e44df9 -> wakeup thread Time-limited test)) after lookup
21:04:12.304 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:12.306 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.306 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.307 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:12.323 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:12.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [4ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [14ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [18ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [34ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@71392203(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@50e7a10 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@65e44df9 -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983))) after lookup
21:04:12.324 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:12.325 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.325 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.326 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.363 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:12.363 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:12.364 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [4ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [14ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [18ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [34ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [35ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [37ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [37ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [74ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@71392203(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@50e7a10 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@65e44df9 -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634))) after lookup
21:04:12.364 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:12.366 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.366 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.366 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.483 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:12.484 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:12.484 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=195), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [4ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [14ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [18ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [34ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [35ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [37ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [37ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [74ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [75ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [77ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [77ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [194ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@71392203(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@50e7a10 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634) -> (continuation of Deferred@732209949 after retry RPC@220353431), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@65e44df9 -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634) -> (continuation of Deferred@732209949 after retry RPC@220353431))) after lookup
21:04:12.485 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:12.487 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.487 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.487 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.644 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:12.644 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:12.645 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=355), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [4ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [14ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [18ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [34ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [35ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [37ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [37ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [74ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [75ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [77ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [77ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [194ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [196ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [198ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [198ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [352ms] refreshing cache from master, [352ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [354ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@71392203(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@50e7a10 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@572062679 after retry RPC@1028357832), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@65e44df9 -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@572062679 after retry RPC@1028357832))) after lookup
21:04:12.645 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:12.646 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.646 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:12.646 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.745970 32433 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.746141 32433 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.746268 32434 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.746371 32434 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.746826 32434 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:13.747440 32257 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.747468 32053 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185" is_pre_election: true
21:04:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.747532 32433 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.747578 32053 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 1.
21:04:13.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.747884 32257 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 
21:04:13.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.748251 32434 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:13.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:13.748225 31987 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:13.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.748302 32434 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:13.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.748337 32434 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 1 FOLLOWER]: Advancing to term 2
21:04:13.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:13.748478 32257 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:13.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:13.749231 31987 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:13.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.749389 32434 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:13.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.749742 32434 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 2 election: Requested vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.750226 32053 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185"
21:04:13.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.750345 32053 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 1 FOLLOWER]: Advancing to term 2
21:04:13.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:13.750757 32257 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:13.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.751268 32053 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 2.
21:04:13.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.751547 32257 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7
21:04:13.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.751684 32434 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Leader election won for term 2
21:04:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.751870 32434 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 LEADER]: Becoming Leader. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Running, Role: LEADER
21:04:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.751881 32323 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.752175 32434 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.752286 31988 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237
21:04:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.752483 32433 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:04:13.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:13.754746 31724 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 reported cstate change: term changed from 1 to 2, leader changed from 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194) to 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195). New cstate: current_term: 2 leader_uuid: "4764cea6067a4b4183944ffb44dfc237" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: HEALTHY } } }
21:04:14.063 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:14.063 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:14.064 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1774), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [4ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [14ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [18ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [34ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [35ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [37ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [37ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [74ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [75ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [77ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [77ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [194ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [196ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [198ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [198ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [352ms] refreshing cache from master, [352ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [354ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [356ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [357ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [357ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1771ms] refreshing cache from master, [1772ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1774ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@71392203(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@50e7a10 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@572062679 after retry RPC@1028357832) -> (continuation of Deferred@854727327 after retry RPC@1518042306), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@65e44df9 -> wakeup thread Time-limited test -> (continuation of Deferred@1411129119 after retry RPC@1921304983) -> (continuation of Deferred@622140374 after retry RPC@1721914634) -> (continuation of Deferred@732209949 after retry RPC@220353431) -> (continuation of Deferred@572062679 after retry RPC@1028357832) -> (continuation of Deferred@854727327 after retry RPC@1518042306))) after lookup
21:04:14.064 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:14.065 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] Successfully connected to peer
21:04:14.084 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x38daaf3d, L:/127.0.0.1:42346 - R:/127.30.240.195:33321] using TOKEN/null
21:04:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.087641 32053 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Refusing update from remote peer 4764cea6067a4b4183944ffb44dfc237: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
21:04:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.088174 32257 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 -> Peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Couldn't send request to peer 42e2bcf086eb4742a41e54464efe56f7. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.088395 32441 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:04:14.104 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.194:37435
21:04:14.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.106449 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.195660 32461 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.195830 32461 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.195878 32461 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.199606 32461 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.199698 32461 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.194
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.202824 32461 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.32461
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.203503 32461 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.204273 32461 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.209395 32467 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.209448 32466 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:14.209393 32469 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.209518 32461 server_base.cc:1061] running on GCE node
21:04:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.210441 32461 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.210889 32461 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.212051 32461 hybrid_clock.cc:648] HybridClock initialized: now 1783544654212044 us; error 27 us; skew 500 ppm
21:04:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.213526 32461 webserver.cc:534] Webserver started at http://127.30.240.194:34591/ using document root <none> and password file <none>
21:04:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.214012 32461 fs_manager.cc:362] Metadata directory not provided
21:04:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.214115 32461 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.217077 32461 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
21:04:14.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.218577 32475 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:14.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.219384 32461 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:14.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.219506 32461 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:14.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.219852 32461 fs_report.cc:389] FS layout report
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/data
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.236964 32461 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.237485 32461 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.237689 32461 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.238174 32461 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.239303 32482 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:14.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.241977 32461 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:14.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.242045 32461 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:14.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.242095 32461 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.243218 32461 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.243283 32461 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:04:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.243497 32482 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap starting.
21:04:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.252614 32461 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.194:37435
21:04:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.252674 32589 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.194:37435 every 8 connection(s)
21:04:14.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.253414 32461 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:14.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.257009 32482 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Log is configured to *not* fsync() on all Append() calls
21:04:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.260828 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 32461
21:04:14.261 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:14.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.263806 32482 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:14.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.265276 32482 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap complete.
21:04:14.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.265801 32591 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.265980 32591 heartbeater.cc:461] Registering TS with master...
21:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.266456 32591 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.266651 32482 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent bootstrapping tablet: real 0.023s	user 0.016s	sys 0.004s
21:04:14.268 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.268028 32590 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.268160 32590 heartbeater.cc:461] Registering TS with master...
21:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.268327 32590 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.268491 31724 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.268688 32482 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.269187 32482 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Initialized, Role: FOLLOWER
21:04:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.269373 31724 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.194:58021
21:04:14.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.269511 31869 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:14.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.269726 32482 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:14.270 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.195:33321
21:04:14.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.270506 32482 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
21:04:14.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:14.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.270972 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 32241
21:04:14.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.271075 32591 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:14.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.272750 32592 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.272938 32592 heartbeater.cc:461] Registering TS with master...
21:04:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.273175 32592 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:14.274322 31797 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:14.279 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] handling channelInactive
21:04:14.279 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:14.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state READY due to: connection closed
21:04:14.280 [DEBUG - Time-limited test] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:14.281 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.281 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.281 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.303 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:14.303 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:14.303 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [23ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1111539744(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@435c39c6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f1eaae6 -> wakeup thread Time-limited test)) after lookup
21:04:14.304 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:14.305 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.305 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.305 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.322 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:14.322 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:14.323 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [23ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [23ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [25ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [25ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [42ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1111539744(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@435c39c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f1eaae6 -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898))) after lookup
21:04:14.323 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:14.324 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.324 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.324 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.363 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:14.363 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:14.363 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=83), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [23ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [23ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [25ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [25ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [42ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [43ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [44ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [44ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [82ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1111539744(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@435c39c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f1eaae6 -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369))) after lookup
21:04:14.363 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:14.364 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.364 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.364 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.443 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:14.443 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:14.443 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=163), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [23ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [23ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [25ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [25ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [42ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [43ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [44ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [44ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [82ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [83ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [84ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [84ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [162ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1111539744(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@435c39c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369) -> (continuation of Deferred@86840144 after retry RPC@1921752459), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f1eaae6 -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369) -> (continuation of Deferred@86840144 after retry RPC@1921752459))) after lookup
21:04:14.444 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:14.444 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.445 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.445 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.643 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:14.643 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:14.644 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=363), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [23ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [23ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [25ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [25ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [42ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [43ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [44ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [44ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [82ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [83ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [84ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [84ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [162ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [163ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [165ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [165ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [361ms] refreshing cache from master, [361ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [363ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1111539744(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@435c39c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369) -> (continuation of Deferred@86840144 after retry RPC@1921752459) -> (continuation of Deferred@465293445 after retry RPC@896692668), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f1eaae6 -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369) -> (continuation of Deferred@86840144 after retry RPC@1921752459) -> (continuation of Deferred@465293445 after retry RPC@896692668))) after lookup
21:04:14.644 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:14.645 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.645 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:14.645 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.604642 32603 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 4764cea6067a4b4183944ffb44dfc237)
21:04:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.604736 32603 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.605074 32603 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:15.606195 31988 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.609483 32544 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7" is_pre_election: true
21:04:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.609709 32544 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6920c90fa92046599c83c12690baf185 in term 1.
21:04:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.610292 31987 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185; no voters: 4764cea6067a4b4183944ffb44dfc237
21:04:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.610478 32603 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:04:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.610569 32603 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 4764cea6067a4b4183944ffb44dfc237)
21:04:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.610613 32603 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 2 FOLLOWER]: Advancing to term 3
21:04:15.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.611410 32603 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:15.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.611786 32603 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 3 election: Requested vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.612051 32544 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.612181 32544 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 1 FOLLOWER]: Advancing to term 3
21:04:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:15.612815 31988 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:15.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.613818 32544 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6920c90fa92046599c83c12690baf185 in term 3.
21:04:15.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.614091 31987 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185; no voters: 4764cea6067a4b4183944ffb44dfc237
21:04:15.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.614236 32603 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Leader election won for term 3
21:04:15.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.614444 32603 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 LEADER]: Becoming Leader. State: Replica: 6920c90fa92046599c83c12690baf185, State: Running, Role: LEADER
21:04:15.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.614740 32603 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:15.617202 31724 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 reported cstate change: term changed from 2 to 3, leader changed from 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195) to 6920c90fa92046599c83c12690baf185 (127.30.240.193). New cstate: current_term: 3 leader_uuid: "6920c90fa92046599c83c12690baf185" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: UNKNOWN } } }
21:04:16.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.059994 31988 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 -> Peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Couldn't send request to peer 4764cea6067a4b4183944ffb44dfc237. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:16.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.162230 32544 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 3 FOLLOWER]: Refusing update from remote peer 6920c90fa92046599c83c12690baf185: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
21:04:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.162838 32603 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
21:04:16.544 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:16.544 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:16.544 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2264), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [23ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [23ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [25ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [25ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [42ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [43ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [44ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [44ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [81ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [82ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [83ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [84ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [84ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [161ms] refreshing cache from master, [161ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [162ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [163ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [165ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [165ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [361ms] refreshing cache from master, [361ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [363ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [364ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [365ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [365ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2261ms] refreshing cache from master, [2261ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [2264ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1111539744(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@435c39c6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369) -> (continuation of Deferred@86840144 after retry RPC@1921752459) -> (continuation of Deferred@465293445 after retry RPC@896692668) -> (continuation of Deferred@1467216159 after retry RPC@2145784760), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f1eaae6 -> wakeup thread Time-limited test -> (continuation of Deferred@1550252296 after retry RPC@997444898) -> (continuation of Deferred@886245060 after retry RPC@300734369) -> (continuation of Deferred@86840144 after retry RPC@1921752459) -> (continuation of Deferred@465293445 after retry RPC@896692668) -> (continuation of Deferred@1467216159 after retry RPC@2145784760))) after lookup
21:04:16.545 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:16.546 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] Successfully connected to peer
21:04:16.562 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9ae55cd4, L:/127.0.0.1:33692 - R:/127.30.240.193:46783] using TOKEN/null
21:04:16.578 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.195:33321
21:04:16.579 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.579560 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:33321
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42855
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:16.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.668300 32631 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.668434 32631 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.668469 32631 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:16.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.671293 32631 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:16.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.671371 32631 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.195
21:04:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.674620 32631 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:33321
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42855
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.32631
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.675422 32631 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:16.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.676455 32631 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.682571 32639 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.682727 32631 server_base.cc:1061] running on GCE node
21:04:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.682554 32638 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:16.682554 32641 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.683313 32631 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.683699 32631 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.684851 32631 hybrid_clock.cc:648] HybridClock initialized: now 1783544656684838 us; error 28 us; skew 500 ppm
21:04:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.686230 32631 webserver.cc:534] Webserver started at http://127.30.240.195:42855/ using document root <none> and password file <none>
21:04:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.686697 32631 fs_manager.cc:362] Metadata directory not provided
21:04:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.686784 32631 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.689347 32631 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
21:04:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.691020 32647 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.691761 32631 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.691865 32631 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.692209 32631 fs_report.cc:389] FS layout report
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/data
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.703367 32631 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.703820 32631 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.703981 32631 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.704456 32631 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:16.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.705384 32654 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:16.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.707834 32631 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:16.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.707893 32631 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:16.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.707942 32631 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:16.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.709069 32631 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:16.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.709126 32631 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:04:16.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.709369 32654 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap starting.
21:04:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.718789 32631 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.195:33321
21:04:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.718818 32761 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.195:33321 every 8 connection(s)
21:04:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.719547 32631 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:16.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.722615 32654 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Log is configured to *not* fsync() on all Append() calls
21:04:16.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.723928 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 32631
21:04:16.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.728468 32762 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.728672 32762 heartbeater.cc:461] Registering TS with master...
21:04:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.729163 32762 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:16.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.730890 31869 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:16.732 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:16.733 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.193:46783
21:04:16.733 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.732429 32654 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.732774 32654 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap complete.
21:04:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.733652 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 31971
21:04:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.733958 32764 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.734047 32764 heartbeater.cc:461] Registering TS with master...
21:04:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.734192 32764 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:16.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.735193 31797 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:16.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.735327 32654 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent bootstrapping tablet: real 0.026s	user 0.013s	sys 0.011s
21:04:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.737318 32763 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.737394 32763 heartbeater.cc:461] Registering TS with master...
21:04:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.737581 32763 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.738283 31724 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.738096 32654 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.738476 32654 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Initialized, Role: FOLLOWER
21:04:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.738853 32654 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.739042 31724 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.195:56655
21:04:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.739648 32654 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.000s
21:04:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:16.740486 32763 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:16.741 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] handling channelInactive
21:04:16.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state READY due to: connection closed
21:04:16.742 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:16.743 [DEBUG - Time-limited test] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:16.744 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.744 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.744 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.763 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:16.763 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:16.763 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [21ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@2145804364(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@223967f6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15217ded -> wakeup thread Time-limited test)) after lookup
21:04:16.764 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:16.765 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.765 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.765 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.783 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:16.783 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:16.783 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [21ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [22ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [23ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [41ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@2145804364(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@223967f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15217ded -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364))) after lookup
21:04:16.784 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:16.784 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.784 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.785 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:16.802 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:16.803 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [21ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [22ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [23ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [41ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [42ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [43ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [60ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@2145804364(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@223967f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15217ded -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404))) after lookup
21:04:16.803 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:16.804 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.804 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.804 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.882 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:16.883 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:16.883 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=140), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [21ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [22ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [23ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [41ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [42ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [43ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [60ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [61ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [62ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [62ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [140ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@2145804364(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@223967f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15217ded -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384))) after lookup
21:04:16.884 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:16.885 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.885 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:16.885 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:17.183 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:17.183 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:17.184 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=441), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [21ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [22ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [23ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [41ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [42ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [43ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [60ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [61ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [62ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [62ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [140ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [141ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [143ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [143ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [438ms] refreshing cache from master, [438ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [440ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@2145804364(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@223967f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15217ded -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909))) after lookup
21:04:17.184 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:17.185 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:17.185 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:17.185 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:17.963935   307 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:17.964166   307 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:17.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:17.965173   307 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:17.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:17.966143 32649 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:17.966980 32649 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:17.968770 32544 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7" is_pre_election: true
21:04:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:17.969197 32649 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237; no voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185
21:04:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:17.969411   307 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
21:04:18.064 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:18.064 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:18.064 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1322), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [21ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [22ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [23ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [41ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [42ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [43ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [60ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [61ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [62ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [62ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [140ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [141ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [143ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [143ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [438ms] refreshing cache from master, [438ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [440ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [442ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [443ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [443ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [1319ms] refreshing cache from master, [1319ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1321ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@2145804364(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@223967f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15217ded -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773))) after lookup
21:04:18.065 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:18.066 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:18.066 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:18.066 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:18.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.078141   312 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 6920c90fa92046599c83c12690baf185)
21:04:18.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.078265   312 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:18.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.079205   312 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:18.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:18.079747 32477 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:18.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:18.080653 32477 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:18.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.083493 32716 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:18.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.083678 32716 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 2.
21:04:18.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.084290 32478 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237; no voters: 6920c90fa92046599c83c12690baf185
21:04:18.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.084542   312 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 3 FOLLOWER]: Leader pre-election won for term 4
21:04:18.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.084595   312 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 6920c90fa92046599c83c12690baf185)
21:04:18.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.084623   312 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 3 FOLLOWER]: Advancing to term 4
21:04:18.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.085387   312 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:18.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.085664   312 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 4 election: Requested vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.086020 32716 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.086148 32716 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 2 FOLLOWER]: Advancing to term 4
21:04:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:18.086642 32477 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:18.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.087587 32716 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 4.
21:04:18.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.087857 32478 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237; no voters: 6920c90fa92046599c83c12690baf185
21:04:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.088043   312 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Leader election won for term 4
21:04:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.088270   312 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 LEADER]: Becoming Leader. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Running, Role: LEADER
21:04:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.088551   312 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.090720 31724 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 reported cstate change: term changed from 3 to 4, leader changed from 6920c90fa92046599c83c12690baf185 (127.30.240.193) to 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194). New cstate: current_term: 4 leader_uuid: "42e2bcf086eb4742a41e54464efe56f7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: UNKNOWN } } }
21:04:18.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:18.608498 32477 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 -> Peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Couldn't send request to peer 6920c90fa92046599c83c12690baf185. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.713620 32716 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 4 FOLLOWER]: Refusing update from remote peer 42e2bcf086eb4742a41e54464efe56f7: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
21:04:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.714242   329 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
21:04:18.964 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:18.964 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:18.964 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2221), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [21ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [22ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [23ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [23ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [41ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [42ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [43ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [60ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [61ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [62ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [62ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [140ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [141ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [143ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [143ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [438ms] refreshing cache from master, [438ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [440ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [442ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [443ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [443ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [1319ms] refreshing cache from master, [1319ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1321ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [1323ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [1324ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [1324ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2219ms] refreshing cache from master, [2219ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [2221ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@2145804364(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@223967f6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773) -> (continuation of Deferred@558970929 after retry RPC@1405117011), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15217ded -> wakeup thread Time-limited test -> (continuation of Deferred@982096341 after retry RPC@609834364) -> (continuation of Deferred@1509323083 after retry RPC@929705404) -> (continuation of Deferred@593479794 after retry RPC@1170081384) -> (continuation of Deferred@471644528 after retry RPC@698642909) -> (continuation of Deferred@946381100 after retry RPC@843580773) -> (continuation of Deferred@558970929 after retry RPC@1405117011))) after lookup
21:04:18.965 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:18.966 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] Successfully connected to peer
21:04:18.982 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9fae50f2, L:/127.0.0.1:37690 - R:/127.30.240.194:37435] using TOKEN/null
21:04:18.997 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.193:46783
21:04:18.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:18.997813 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:46783
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37005
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.087306   343 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.087443   343 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.087478   343 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:19.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.090277   343 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:19.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.090370   343 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.193
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.093621   343 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:46783
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37005
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.343
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.094290   343 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.095234   343 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.100296   349 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.100301   351 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.100519   343 server_base.cc:1061] running on GCE node
21:04:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:19.100301   348 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:19.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.101076   343 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:19.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.101567   343 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.102777   343 hybrid_clock.cc:648] HybridClock initialized: now 1783544659102721 us; error 64 us; skew 500 ppm
21:04:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.109786   343 webserver.cc:534] Webserver started at http://127.30.240.193:37005/ using document root <none> and password file <none>
21:04:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.110307   343 fs_manager.cc:362] Metadata directory not provided
21:04:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.110411   343 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:19.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.113775   343 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.000s	sys 0.003s
21:04:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.115316   357 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.116210   343 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.001s
21:04:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.116315   343 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6920c90fa92046599c83c12690baf185"
21:04:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.116732   343 fs_report.cc:389] FS layout report
21:04:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/data
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.141430   343 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.141819   343 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.141974   343 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.142392   343 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.143273   365 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.145718   343 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.145771   343 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.145808   343 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.146873   343 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.146910   343 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:04:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.147202   365 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap starting.
21:04:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.156454   343 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.193:46783
21:04:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.156522   472 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.193:46783 every 8 connection(s)
21:04:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.157261   343 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.162026   365 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Log is configured to *not* fsync() on all Append() calls
21:04:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.162425 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 343
21:04:19.162 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.165650   474 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.165812   474 heartbeater.cc:461] Registering TS with master...
21:04:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.166582   474 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.168113 31725 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.169013 31725 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.193:56131
21:04:19.172 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.173846   473 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.173938   473 heartbeater.cc:461] Registering TS with master...
21:04:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.174073   473 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.174268   365 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.174577   365 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap complete.
21:04:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.174713   475 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.174772   475 heartbeater.cc:461] Registering TS with master...
21:04:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.174865 31869 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.174872   475 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.175804 31797 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.175858   365 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent bootstrapping tablet: real 0.029s	user 0.012s	sys 0.011s
21:04:19.178 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.194:37435
21:04:19.178 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.177943   365 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.178274   365 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6920c90fa92046599c83c12690baf185, State: Initialized, Role: FOLLOWER
21:04:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.178550 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 32461
21:04:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.178658   365 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.179454   365 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent starting tablet: real 0.004s	user 0.006s	sys 0.001s
21:04:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:19.180032   474 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:19.186 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:19.186 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] handling channelInactive
21:04:19.187 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state READY due to: connection closed
21:04:19.201 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:19.203 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.203 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.203 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.224 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:19.224 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:19.224 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [17ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2b9d8b27 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test)) after lookup
21:04:19.225 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:19.226 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.226 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.226 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.242 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:19.242 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:19.243 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [17ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [39ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [40ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [40ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874))) after lookup
21:04:19.243 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:19.244 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.244 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.244 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.303 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:19.303 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:19.304 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=117), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [17ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [39ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [40ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [40ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [58ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [58ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [117ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115))) after lookup
21:04:19.304 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:19.305 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.305 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.305 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.363 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:19.363 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:19.363 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=176), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [17ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [39ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [40ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [40ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [58ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [58ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [117ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [118ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [119ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [119ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [176ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021))) after lookup
21:04:19.363 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:19.364 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.364 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.365 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.523 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:19.523 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:19.524 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=337), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [17ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [39ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [40ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [40ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [58ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [58ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [117ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [118ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [119ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [119ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [176ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [177ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [178ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [179ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [335ms] refreshing cache from master, [335ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [337ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016))) after lookup
21:04:19.525 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:19.526 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.526 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:19.526 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:20.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.477324   486 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:20.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.477473   486 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:20.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.478430   486 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.478996   360 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.479832   360 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.482038 32716 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.482486   361 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237
21:04:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.482722   486 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:04:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.497304   491 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.497401   491 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.497723   491 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:20.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.498914 32649 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.501798   427 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185" is_pre_election: true
21:04:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.501981   427 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 3.
21:04:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.502496 32649 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7
21:04:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.502653   491 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 4 FOLLOWER]: Leader pre-election won for term 5
21:04:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.502723   491 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.502763   491 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 4 FOLLOWER]: Advancing to term 5
21:04:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.503453   491 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.503765   491 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 5 election: Requested vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.504266   427 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185"
21:04:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.504384   427 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 3 FOLLOWER]: Advancing to term 5
21:04:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.504693 32649 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:20.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.505877   427 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 5.
21:04:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.506153 32649 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7
21:04:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.506319   491 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 FOLLOWER]: Leader election won for term 5
21:04:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.506541   491 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 LEADER]: Becoming Leader. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Running, Role: LEADER
21:04:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.506824   491 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:20.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.508906 31725 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 reported cstate change: term changed from 4 to 5, leader changed from 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194) to 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195). New cstate: current_term: 5 leader_uuid: "4764cea6067a4b4183944ffb44dfc237" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: HEALTHY } } }
21:04:20.843 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:20.843 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:20.844 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1657), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [17ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [17ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [39ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [40ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [40ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [58ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [58ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [115ms] refreshing cache from master, [115ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [117ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [118ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [119ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [119ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [176ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [177ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [178ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [179ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [335ms] refreshing cache from master, [335ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [337ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [338ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [340ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [340ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1655ms] refreshing cache from master, [1655ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1657ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@563575833(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2b9d8b27 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016) -> (continuation of Deferred@1840925447 after retry RPC@486299108), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2de4b423 -> wakeup thread Time-limited test -> (continuation of Deferred@788327250 after retry RPC@649135874) -> (continuation of Deferred@464120379 after retry RPC@898529115) -> (continuation of Deferred@1509251551 after retry RPC@520608021) -> (continuation of Deferred@1903858731 after retry RPC@1351890016) -> (continuation of Deferred@1840925447 after retry RPC@486299108))) after lookup
21:04:20.845 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:20.846 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] Successfully connected to peer
21:04:20.868 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x7ebd5509, L:/127.0.0.1:41042 - R:/127.30.240.195:33321] using TOKEN/null
21:04:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.872303   427 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 5 FOLLOWER]: Refusing update from remote peer 4764cea6067a4b4183944ffb44dfc237: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 18. (index mismatch)
21:04:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.872852 32649 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 -> Peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Couldn't send request to peer 42e2bcf086eb4742a41e54464efe56f7. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:20.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.873113   496 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
21:04:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.877562   509 mvcc.cc:204] Tried to move back new op lower bound from 7305398930926837760 to 7305398929438179328. Current Snapshot: MvccSnapshot[applied={T|T < 7305398930926837760}]
21:04:20.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.881335   510 mvcc.cc:204] Tried to move back new op lower bound from 7305398930926837760 to 7305398929438179328. Current Snapshot: MvccSnapshot[applied={T|T < 7305398930926837760}]
21:04:20.893 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.194:37435
21:04:20.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.894179 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:20.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.984886   517 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.985023   517 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.985065   517 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.987660   517 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.987737   517 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.194
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.991356   517 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.517
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:20.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.992054   517 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:20.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.993078   517 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.999186   525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.999187   522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.999473   517 server_base.cc:1061] running on GCE node
21:04:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:20.999181   523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:21.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:20.999912   517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:21.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.000470   517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.001631   517 hybrid_clock.cc:648] HybridClock initialized: now 1783544661001617 us; error 30 us; skew 500 ppm
21:04:21.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.003005   517 webserver.cc:534] Webserver started at http://127.30.240.194:34591/ using document root <none> and password file <none>
21:04:21.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.003473   517 fs_manager.cc:362] Metadata directory not provided
21:04:21.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.003548   517 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:21.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.006145   517 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.001s	sys 0.000s
21:04:21.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.007627   531 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.008476   517 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.008566   517 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.008821   517 fs_report.cc:389] FS layout report
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/data
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:21.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.026371   517 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.026754   517 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.026860   517 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.027206   517 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.028075   538 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.030809   517 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.030864   517 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.030911   517 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.032330   517 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.032377   517 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
21:04:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.032675   538 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap starting.
21:04:21.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.041301   517 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.194:37435
21:04:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.042289   517 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:21.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.042778   645 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.194:37435 every 8 connection(s)
21:04:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.046926   538 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Log is configured to *not* fsync() on all Append() calls
21:04:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.048498 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 517
21:04:21.048 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.055291   648 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.055474   646 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.055480   648 heartbeater.cc:461] Registering TS with master...
21:04:21.055 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.055784   646 heartbeater.cc:461] Registering TS with master...
21:04:21.056 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.195:33321
21:04:21.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.056150   648 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.056154   646 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.057230   647 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.057315   647 heartbeater.cc:461] Registering TS with master...
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.057451 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 32631
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.057482   647 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.057781 31868 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.058246 31797 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.059115 31724 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.059989 31724 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.194:59757
21:04:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.061436   538 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.061750   538 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap complete.
21:04:21.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.062852   538 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent bootstrapping tablet: real 0.030s	user 0.020s	sys 0.006s
21:04:21.065 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:21.065 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] handling channelInactive
21:04:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.065295   538 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.065649   538 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Initialized, Role: FOLLOWER
21:04:21.065 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state READY due to: connection closed
21:04:21.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.066056   538 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:21.066 [DEBUG - Time-limited test] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:21.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.066757   538 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent starting tablet: real 0.004s	user 0.006s	sys 0.000s
21:04:21.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:21.066926   647 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:21.068 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.068 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.068 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.082 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:21.083 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:21.083 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [2ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [16ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@814068044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2c437654 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@630c9858 -> wakeup thread Time-limited test)) after lookup
21:04:21.083 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:21.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.084 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.084 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.103 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:21.103 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:21.104 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [2ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [16ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [17ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [18ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@814068044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2c437654 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@630c9858 -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975))) after lookup
21:04:21.104 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:21.105 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.105 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.105 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:21.122 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:21.123 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [2ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [16ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [17ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [18ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@814068044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2c437654 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@630c9858 -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540))) after lookup
21:04:21.123 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:21.124 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.124 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.124 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.223 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:21.223 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:21.223 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=157), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [2ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [16ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [17ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [18ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [58ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [58ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [154ms] refreshing cache from master, [155ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [156ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@814068044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2c437654 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@630c9858 -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519))) after lookup
21:04:21.224 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:21.224 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.224 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.224 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.523 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:21.523 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:21.524 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=458), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [2ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [16ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [17ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [18ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [58ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [58ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [154ms] refreshing cache from master, [155ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [156ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [157ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [158ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [158ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [454ms] refreshing cache from master, [455ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [456ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@814068044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2c437654 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@630c9858 -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319))) after lookup
21:04:21.524 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:21.525 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.526 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:21.526 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:22.123 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:22.123 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:22.124 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1058), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [2ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [16ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [17ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [18ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [58ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [58ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [154ms] refreshing cache from master, [155ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [156ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [157ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [158ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [158ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [454ms] refreshing cache from master, [455ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [456ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [458ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [460ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [460ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1055ms] refreshing cache from master, [1055ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1057ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@814068044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2c437654 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@630c9858 -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564))) after lookup
21:04:22.124 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:22.125 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:22.125 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:22.125 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.202044   659 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.202185   659 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:22.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.203116   659 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:22.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:22.203768   534 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:22.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:22.204586   534 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.206578   427 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185" is_pre_election: true
21:04:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.207023   533 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7; no voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185
21:04:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.207286   659 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:04:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.393666   664 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 4764cea6067a4b4183944ffb44dfc237)
21:04:22.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.393779   664 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:22.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.394196   664 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:22.395421   361 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:22.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.398070   599 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7" is_pre_election: true
21:04:22.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.398240   599 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6920c90fa92046599c83c12690baf185 in term 4.
21:04:22.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.398790   360 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185; no voters: 4764cea6067a4b4183944ffb44dfc237
21:04:22.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.398949   664 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 5 FOLLOWER]: Leader pre-election won for term 6
21:04:22.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.399008   664 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 4764cea6067a4b4183944ffb44dfc237)
21:04:22.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.399042   664 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 5 FOLLOWER]: Advancing to term 6
21:04:22.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.399703   664 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:22.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.400003   664 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 6 election: Requested vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:22.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.400355   599 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:22.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.400485   599 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 4 FOLLOWER]: Advancing to term 6
21:04:22.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:22.401026   361 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.402081   599 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6920c90fa92046599c83c12690baf185 in term 6.
21:04:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.402367   360 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185; no voters: 4764cea6067a4b4183944ffb44dfc237
21:04:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.402505   664 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Leader election won for term 6
21:04:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.402655   664 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 LEADER]: Becoming Leader. State: Replica: 6920c90fa92046599c83c12690baf185, State: Running, Role: LEADER
21:04:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.403121   664 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.405364 31724 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 reported cstate change: term changed from 5 to 6, leader changed from 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195) to 6920c90fa92046599c83c12690baf185 (127.30.240.193). New cstate: current_term: 6 leader_uuid: "6920c90fa92046599c83c12690baf185" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: UNKNOWN } } }
21:04:22.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.812932   599 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Refusing update from remote peer 6920c90fa92046599c83c12690baf185: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
21:04:22.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:22.813565   664 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
21:04:22.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:22.821177   361 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 -> Peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Couldn't send request to peer 4764cea6067a4b4183944ffb44dfc237. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:24.803 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:24.804 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:24.804 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=3738), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [2ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [2ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [16ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [17ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [18ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [18ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [56ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [57ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [58ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [58ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [154ms] refreshing cache from master, [155ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [156ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [157ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [158ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [158ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [454ms] refreshing cache from master, [455ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [456ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [458ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [460ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [460ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1055ms] refreshing cache from master, [1055ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1057ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [1058ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1059ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1059ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [3735ms] refreshing cache from master, [3735ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [3737ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@814068044(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@2c437654 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564) -> (continuation of Deferred@337021542 after retry RPC@1478290429), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@630c9858 -> wakeup thread Time-limited test -> (continuation of Deferred@296394690 after retry RPC@375593975) -> (continuation of Deferred@1478778548 after retry RPC@1444301540) -> (continuation of Deferred@903347627 after retry RPC@1374863519) -> (continuation of Deferred@1572959056 after retry RPC@443594319) -> (continuation of Deferred@162940939 after retry RPC@384517564) -> (continuation of Deferred@337021542 after retry RPC@1478290429))) after lookup
21:04:24.805 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:24.806 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] Successfully connected to peer
21:04:24.821 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x40f5f9f1, L:/127.0.0.1:42058 - R:/127.30.240.193:46783] using TOKEN/null
21:04:24.840 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.195:33321
21:04:24.841 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.841991 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:33321
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42855
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.928357   706 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.928489   706 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.928524   706 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.931473   706 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.931595   706 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.195
21:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.934777   706 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:33321
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42855
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.706
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.935405   706 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:24.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.936175   706 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.941148   714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.941131   711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:24.941174   712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.941270   706 server_base.cc:1061] running on GCE node
21:04:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.941854   706 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.942284   706 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.943432   706 hybrid_clock.cc:648] HybridClock initialized: now 1783544664943417 us; error 26 us; skew 500 ppm
21:04:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.944743   706 webserver.cc:534] Webserver started at http://127.30.240.195:42855/ using document root <none> and password file <none>
21:04:24.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.945178   706 fs_manager.cc:362] Metadata directory not provided
21:04:24.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.945248   706 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.947713   706 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
21:04:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.949318   720 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.950002   706 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.950103   706 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.950453   706 fs_report.cc:389] FS layout report
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/data
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:24.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.960227   706 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:24.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.960747   706 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.960922   706 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.961406   706 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.962352   727 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.964867   706 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.964928   706 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.964977   706 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:24.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.966051   706 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:24.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.966111   706 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:04:24.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.966347   727 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap starting.
21:04:24.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.975826   706 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.195:33321
21:04:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.975915   834 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.195:33321 every 8 connection(s)
21:04:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.976691   706 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.981107   727 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Log is configured to *not* fsync() on all Append() calls
21:04:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.984970   837 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.985167   837 heartbeater.cc:461] Registering TS with master...
21:04:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.985495   836 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.985563   836 heartbeater.cc:461] Registering TS with master...
21:04:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.985632   837 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.985710   836 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.986436 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 706
21:04:24.987 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.986824   835 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.986886   835 heartbeater.cc:461] Registering TS with master...
21:04:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.987022   835 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.987165 31724 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.987534 31797 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.987910 31724 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.195:36309
21:04:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.988611 31869 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:24.994 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:24.994 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.193:46783
21:04:24.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.995206 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 343
21:04:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.996624   727 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.996939   727 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap complete.
21:04:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.997834   727 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent bootstrapping tablet: real 0.032s	user 0.025s	sys 0.004s
21:04:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.999399   727 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:24.999661   727 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Initialized, Role: FOLLOWER
21:04:25.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.000010   727 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.000735   727 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent starting tablet: real 0.003s	user 0.004s	sys 0.000s
21:04:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:25.001258   836 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:25.002 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] handling channelInactive
21:04:25.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:25.002 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state READY due to: connection closed
21:04:25.003 [DEBUG - Time-limited test] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:25.003 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.004 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.004 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:25.023 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:25.023 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [20ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test)) after lookup
21:04:25.023 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:25.024 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.024 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.043 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:25.043 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:25.043 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [20ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [21ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [22ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [40ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965))) after lookup
21:04:25.044 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:25.044 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.044 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.045 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.083 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:25.083 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:25.084 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [20ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [21ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [22ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [40ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [41ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [42ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [81ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644))) after lookup
21:04:25.084 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:25.085 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.085 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.085 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.223 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:25.224 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:25.224 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=221), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [20ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [21ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [22ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [40ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [41ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [42ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [81ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [82ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [83ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [219ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [221ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021))) after lookup
21:04:25.224 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:25.225 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.225 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.225 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.704 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:25.705 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:25.705 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=702), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [20ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [21ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [22ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [40ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [41ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [42ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [81ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [82ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [83ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [219ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [221ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [222ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [223ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [223ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [699ms] refreshing cache from master, [699ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [702ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138))) after lookup
21:04:25.706 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:25.707 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.707 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:25.707 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:26.083 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:26.084 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:26.084 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1081), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [20ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [21ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [22ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [40ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [41ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [42ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [81ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [82ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [83ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [219ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [221ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [222ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [223ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [223ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [699ms] refreshing cache from master, [699ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [702ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [703ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [705ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [705ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [1079ms] refreshing cache from master, [1079ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1081ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223))) after lookup
21:04:26.085 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:26.086 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:26.086 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:26.086 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:26.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.335891   848 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:26.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.336073   848 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:26.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.337064   848 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.337877   722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.338608   722 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.340319   853 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 6920c90fa92046599c83c12690baf185)
21:04:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.340396   853 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.340401   600 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7" is_pre_election: true
21:04:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.340623   600 raft_consensus.cc:2393] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 4764cea6067a4b4183944ffb44dfc237 in current term 6: Already voted for candidate 6920c90fa92046599c83c12690baf185 in this term.
21:04:26.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.340746   853 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:26.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.341109   722 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237; no voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185
21:04:26.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.341351   848 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 5 FOLLOWER]: Advancing to term 6
21:04:26.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.342337   533 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:26.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.343053   848 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 6 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
21:04:26.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.344936   789 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:26.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.345118   789 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 6.
21:04:26.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.345696   534 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237; no voters: 6920c90fa92046599c83c12690baf185
21:04:26.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.345850   853 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Leader pre-election won for term 7
21:04:26.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.345924   853 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 6920c90fa92046599c83c12690baf185)
21:04:26.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.345954   853 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 6 FOLLOWER]: Advancing to term 7
21:04:26.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.346608   853 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:26.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.346889   853 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 7 election: Requested vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:26.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.347189   789 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:26.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.347332   789 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 6 FOLLOWER]: Advancing to term 7
21:04:26.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.347764   533 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:26.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.348109   789 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 7.
21:04:26.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.348413   534 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237; no voters: 6920c90fa92046599c83c12690baf185
21:04:26.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.348574   853 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 7 FOLLOWER]: Leader election won for term 7
21:04:26.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.348712   853 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 7 LEADER]: Becoming Leader. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Running, Role: LEADER
21:04:26.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.348973   853 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.350903 31724 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 reported cstate change: term changed from 6 to 7, leader changed from 6920c90fa92046599c83c12690baf185 (127.30.240.193) to 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194). New cstate: current_term: 7 leader_uuid: "42e2bcf086eb4742a41e54464efe56f7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: UNKNOWN } } }
21:04:26.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:26.810530   533 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 -> Peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Couldn't send request to peer 6920c90fa92046599c83c12690baf185. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.944972   789 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 7 FOLLOWER]: Refusing update from remote peer 42e2bcf086eb4742a41e54464efe56f7: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
21:04:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:26.945531   853 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
21:04:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:27.209184   533 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:04:28.384 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:28.384 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:28.384 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=3382), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [18ms] refreshing cache from master, [18ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [20ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [21ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [22ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [40ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [41ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [42ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [43ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [81ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [82ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [83ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [219ms] refreshing cache from master, [219ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [221ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [222ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [223ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [223ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [699ms] refreshing cache from master, [699ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [702ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [703ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [705ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [705ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [1079ms] refreshing cache from master, [1079ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1081ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [1082ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [1084ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [1084ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [3379ms] refreshing cache from master, [3379ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [3381ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@231899643(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@268a1c31 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223) -> (continuation of Deferred@673225744 after retry RPC@674373503), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@21195f8 -> wakeup thread Time-limited test -> (continuation of Deferred@344128144 after retry RPC@447873965) -> (continuation of Deferred@616621778 after retry RPC@786773644) -> (continuation of Deferred@1614865895 after retry RPC@1729556021) -> (continuation of Deferred@1519042504 after retry RPC@171804138) -> (continuation of Deferred@1224815344 after retry RPC@804650223) -> (continuation of Deferred@673225744 after retry RPC@674373503))) after lookup
21:04:28.385 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:28.386 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] Successfully connected to peer
21:04:28.400 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x6ffc3572, L:/127.0.0.1:55862 - R:/127.30.240.194:37435] using TOKEN/null
21:04:28.414 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.193:46783
21:04:28.414 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.415014 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:46783
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37005
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.502271   888 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.502405   888 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.502439   888 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.505257   888 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.505347   888 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.193
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.508419   888 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:46783
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37005
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.888
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.509188   888 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.509961   888 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.515022   896 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.515022   894 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:28.515097   893 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.515441   888 server_base.cc:1061] running on GCE node
21:04:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.515753   888 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.516291   888 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:28.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.517483   888 hybrid_clock.cc:648] HybridClock initialized: now 1783544668517468 us; error 21 us; skew 500 ppm
21:04:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.518759   888 webserver.cc:534] Webserver started at http://127.30.240.193:37005/ using document root <none> and password file <none>
21:04:28.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.519153   888 fs_manager.cc:362] Metadata directory not provided
21:04:28.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.519197   888 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:28.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.521799   888 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.001s	sys 0.001s
21:04:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.523541   902 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.524369   888 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.524441   888 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6920c90fa92046599c83c12690baf185"
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.524691   888 fs_report.cc:389] FS layout report
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/data
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.554718   888 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.555109   888 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.555219   888 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.555563   888 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:28.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.556476   911 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.558965   888 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.559036   888 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.559084   888 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.560524   888 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.560573   888 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
21:04:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.560864   911 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap starting.
21:04:28.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.570133   888 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.193:46783
21:04:28.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.570487  1018 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.193:46783 every 8 connection(s)
21:04:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.571022   888 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.575955   911 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Log is configured to *not* fsync() on all Append() calls
21:04:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.578955  1020 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.579339  1019 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.579427  1019 heartbeater.cc:461] Registering TS with master...
21:04:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.579424 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 888
21:04:28.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.579869  1019 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.581197  1020 heartbeater.cc:461] Registering TS with master...
21:04:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.581413  1020 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.582211 31724 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.582424 31869 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.582919 31724 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.193:60553
21:04:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.585893  1021 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.586088  1021 heartbeater.cc:461] Registering TS with master...
21:04:28.586 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.586335  1021 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:28.586 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.194:37435
21:04:28.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.587317 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 517
21:04:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.588843 31795 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:28.594 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] handling channelInactive
21:04:28.595 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state READY due to: connection closed
21:04:28.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:28.596 [DEBUG - Time-limited test] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:28.596 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.597 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.597 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.597823   911 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:28.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.598078   911 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap complete.
21:04:28.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.598857   911 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent bootstrapping tablet: real 0.038s	user 0.026s	sys 0.005s
21:04:28.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.600381   911 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:28.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.600628   911 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6920c90fa92046599c83c12690baf185, State: Initialized, Role: FOLLOWER
21:04:28.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.600986   911 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.601786   911 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent starting tablet: real 0.003s	user 0.005s	sys 0.000s
21:04:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:28.602258  1020 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:28.603 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:28.603 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:28.603 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=7), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [2ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [2ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [8ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@665651085(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@76c3f651 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d5c245b -> wakeup thread Time-limited test)) after lookup
21:04:28.604 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:28.604 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.604 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.604 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.623 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:28.623 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:28.623 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [2ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [2ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [8ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [8ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [9ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [10ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@665651085(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@76c3f651 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d5c245b -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995))) after lookup
21:04:28.623 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:28.624 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.624 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.624 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.642 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:28.642 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:28.643 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [2ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [2ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [8ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [8ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [9ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [10ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [28ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [29ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [47ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@665651085(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@76c3f651 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d5c245b -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852))) after lookup
21:04:28.643 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:28.644 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.644 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.644 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.703 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:28.704 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:28.704 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=108), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [2ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [2ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [8ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [8ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [9ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [10ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [28ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [29ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [47ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [48ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [49ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [49ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [108ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@665651085(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@76c3f651 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852) -> (continuation of Deferred@1183865992 after retry RPC@228391236), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d5c245b -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852) -> (continuation of Deferred@1183865992 after retry RPC@228391236))) after lookup
21:04:28.704 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:28.705 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.705 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:28.705 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:29.024 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:29.025 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:29.025 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=429), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [2ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [2ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [8ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [8ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [9ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [10ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [28ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [29ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [47ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [48ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [49ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [49ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [108ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [109ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [110ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [110ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [426ms] refreshing cache from master, [426ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [429ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@665651085(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@76c3f651 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852) -> (continuation of Deferred@1183865992 after retry RPC@228391236) -> (continuation of Deferred@409502650 after retry RPC@1918111530), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d5c245b -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852) -> (continuation of Deferred@1183865992 after retry RPC@228391236) -> (continuation of Deferred@409502650 after retry RPC@1918111530))) after lookup
21:04:29.026 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:29.026 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:29.027 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:29.027 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:29.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.892220  1032 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:29.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.892370  1032 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:29.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.893333  1032 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.893968   904 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.894732   904 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.896930   789 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.897392   905 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237
21:04:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.897622  1032 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
21:04:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.914731  1037 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.914809  1037 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.915069  1037 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.916211   722 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.918936   973 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185" is_pre_election: true
21:04:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.919080   973 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 6.
21:04:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.919548   722 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7
21:04:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.919672  1037 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 7 FOLLOWER]: Leader pre-election won for term 8
21:04:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.919726  1037 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.919752  1037 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 7 FOLLOWER]: Advancing to term 8
21:04:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.920442  1037 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:29.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.920707  1037 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 8 election: Requested vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.921182   973 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185"
21:04:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.921303   973 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 6 FOLLOWER]: Advancing to term 8
21:04:29.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:29.921527   722 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.922883   973 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 8.
21:04:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.923147   722 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7
21:04:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.923314  1037 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 8 FOLLOWER]: Leader election won for term 8
21:04:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.923496  1037 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 8 LEADER]: Becoming Leader. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Running, Role: LEADER
21:04:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.923772  1037 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:29.925761 31724 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 reported cstate change: term changed from 7 to 8, leader changed from 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194) to 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195). New cstate: current_term: 8 leader_uuid: "4764cea6067a4b4183944ffb44dfc237" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: HEALTHY } } }
21:04:30.183 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:30.184 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:30.184 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1588), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [2ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [2ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [8ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [8ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [9ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [10ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [28ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [29ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [47ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [48ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [49ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [49ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [108ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [109ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [110ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [110ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [426ms] refreshing cache from master, [426ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [429ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [430ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [432ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [432ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1586ms] refreshing cache from master, [1586ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1588ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@665651085(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@76c3f651 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852) -> (continuation of Deferred@1183865992 after retry RPC@228391236) -> (continuation of Deferred@409502650 after retry RPC@1918111530) -> (continuation of Deferred@539378024 after retry RPC@1926833945), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d5c245b -> wakeup thread Time-limited test -> (continuation of Deferred@1638575817 after retry RPC@1834491995) -> (continuation of Deferred@1642488955 after retry RPC@1301338852) -> (continuation of Deferred@1183865992 after retry RPC@228391236) -> (continuation of Deferred@409502650 after retry RPC@1918111530) -> (continuation of Deferred@539378024 after retry RPC@1926833945))) after lookup
21:04:30.185 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:30.185 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] Successfully connected to peer
21:04:30.207 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x8e376c9d, L:/127.0.0.1:41096 - R:/127.30.240.195:33321] using TOKEN/null
21:04:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.210181   973 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 8 FOLLOWER]: Refusing update from remote peer 4764cea6067a4b4183944ffb44dfc237: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
21:04:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.210455   722 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 -> Peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Couldn't send request to peer 42e2bcf086eb4742a41e54464efe56f7. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.210739  1042 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
21:04:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.214026  1056 mvcc.cc:204] Tried to move back new op lower bound from 7305398961777926144 to 7305398953367281664. Current Snapshot: MvccSnapshot[applied={T|T < 7305398961777926144}]
21:04:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.220455  1055 mvcc.cc:204] Tried to move back new op lower bound from 7305398969175363584 to 7305398968010076160. Current Snapshot: MvccSnapshot[applied={T|T < 7305398969175363584 or (T in {7305398969175363584})}]
21:04:30.229 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.194:37435
21:04:30.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.230566 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.319427  1063 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.319607  1063 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.319657  1063 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.322731  1063 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.322839  1063 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.194
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.325986  1063 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1063
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.326915  1063 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.327755  1063 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.332901  1069 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.332897  1071 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:30.332922  1068 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.333191  1063 server_base.cc:1061] running on GCE node
21:04:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.333462  1063 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.333871  1063 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:30.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.335000  1063 hybrid_clock.cc:648] HybridClock initialized: now 1783544670334985 us; error 29 us; skew 500 ppm
21:04:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.336364  1063 webserver.cc:534] Webserver started at http://127.30.240.194:34591/ using document root <none> and password file <none>
21:04:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.336742  1063 fs_manager.cc:362] Metadata directory not provided
21:04:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.336786  1063 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.339237  1063 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
21:04:30.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.340694  1077 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:30.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.341590  1063 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.341670  1063 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.341943  1063 fs_report.cc:389] FS layout report
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/data
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:30.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.351625  1063 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.352048  1063 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.352233  1063 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.352617  1063 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:30.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.353502  1084 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:30.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.355973  1063 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:30.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.356019  1063 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:30.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.356053  1063 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:30.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.357096  1063 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:30.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.357132  1063 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:04:30.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.357411  1084 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap starting.
21:04:30.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.365808  1063 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.194:37435
21:04:30.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.365871  1191 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.194:37435 every 8 connection(s)
21:04:30.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.366643  1063 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:30.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.369053 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 1063
21:04:30.369 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:30.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.373674  1084 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Log is configured to *not* fsync() on all Append() calls
21:04:30.376 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.376935  1192 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.377139  1192 heartbeater.cc:461] Registering TS with master...
21:04:30.377 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.195:33321
21:04:30.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.377604  1192 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.377650 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 706
21:04:30.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.378764  1193 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:30.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.378841  1193 heartbeater.cc:461] Registering TS with master...
21:04:30.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.379010  1193 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:30.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.379786 31724 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:30.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.380510 31724 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.194:54151
21:04:30.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.380609 31869 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:30.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.382757  1194 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:30.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.382829  1194 heartbeater.cc:461] Registering TS with master...
21:04:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.382943  1194 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.383692 31795 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:30.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:30.386 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] handling channelInactive
21:04:30.386 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state READY due to: connection closed
21:04:30.386 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: connection closed
21:04:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.397452  1084 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.397782  1084 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap complete.
21:04:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.398639  1084 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent bootstrapping tablet: real 0.041s	user 0.016s	sys 0.014s
21:04:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.400405  1084 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.400697  1084 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Initialized, Role: FOLLOWER
21:04:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.401103  1084 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.402060  1193 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:30.402752  1084 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent starting tablet: real 0.004s	user 0.005s	sys 0.000s
21:04:30.403 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:30.404 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:30.404 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [18ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1338189423(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@74992161 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3e7270 -> wakeup thread Time-limited test)) after lookup
21:04:30.404 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:30.405 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.405 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.405 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.422 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:30.422 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:30.423 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [18ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [19ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [20ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [20ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1338189423(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@74992161 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3e7270 -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239))) after lookup
21:04:30.423 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:30.424 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.424 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.424 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.462 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:30.463 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:30.463 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [18ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [19ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [20ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [20ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [77ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1338189423(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@74992161 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3e7270 -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086))) after lookup
21:04:30.463 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:30.464 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.464 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.464 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.522 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:30.522 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:30.523 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=137), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [18ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [19ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [20ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [20ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [77ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [78ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [79ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [79ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [135ms] refreshing cache from master, [135ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [137ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1338189423(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@74992161 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086) -> (continuation of Deferred@361723216 after retry RPC@362569229), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3e7270 -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086) -> (continuation of Deferred@361723216 after retry RPC@362569229))) after lookup
21:04:30.523 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:30.524 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.524 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.524 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.844 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:30.844 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:30.844 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=459), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [18ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [19ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [20ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [20ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [77ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [78ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [79ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [79ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [135ms] refreshing cache from master, [135ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [137ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [138ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [139ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [139ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [456ms] refreshing cache from master, [456ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [458ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1338189423(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@74992161 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086) -> (continuation of Deferred@361723216 after retry RPC@362569229) -> (continuation of Deferred@483920214 after retry RPC@1807958262), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3e7270 -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086) -> (continuation of Deferred@361723216 after retry RPC@362569229) -> (continuation of Deferred@483920214 after retry RPC@1807958262))) after lookup
21:04:30.845 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:30.846 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.846 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state CONNECTING due to: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:30.846 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321
21:04:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.729997  1206 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 4764cea6067a4b4183944ffb44dfc237)
21:04:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.730129  1206 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.730608  1206 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:31.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.732002   905 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:31.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.734977  1146 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7" is_pre_election: true
21:04:31.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.735199  1146 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6920c90fa92046599c83c12690baf185 in term 7.
21:04:31.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.735807   904 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185; no voters: 4764cea6067a4b4183944ffb44dfc237
21:04:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.735986  1206 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 8 FOLLOWER]: Leader pre-election won for term 9
21:04:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.736059  1206 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 4764cea6067a4b4183944ffb44dfc237)
21:04:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.736106  1206 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 8 FOLLOWER]: Advancing to term 9
21:04:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.742097  1206 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.742442  1206 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 9 election: Requested vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.742767  1146 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.742882  1146 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 7 FOLLOWER]: Advancing to term 9
21:04:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:31.743307   905 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111)
21:04:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.744444  1146 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6920c90fa92046599c83c12690baf185 in term 9.
21:04:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.744693   904 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 6920c90fa92046599c83c12690baf185; no voters: 4764cea6067a4b4183944ffb44dfc237
21:04:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.744876  1206 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Leader election won for term 9
21:04:31.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.745029  1206 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 LEADER]: Becoming Leader. State: Replica: 6920c90fa92046599c83c12690baf185, State: Running, Role: LEADER
21:04:31.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.745292  1206 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:31.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:31.747239 31725 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 reported cstate change: term changed from 8 to 9, leader changed from 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195) to 6920c90fa92046599c83c12690baf185 (127.30.240.193). New cstate: current_term: 9 leader_uuid: "6920c90fa92046599c83c12690baf185" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: UNKNOWN } } }
21:04:32.024 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:32.024 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:32.024 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1639), Traces: [0ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [1ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [18ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [19ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [20ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [20ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [37ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [38ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [39ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [39ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [77ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [78ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [79ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [79ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [135ms] refreshing cache from master, [135ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [137ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [138ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [139ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [139ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [456ms] refreshing cache from master, [456ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [458ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [459ms] sending RPC to server 4764cea6067a4b4183944ffb44dfc237, [461ms] received response from server 4764cea6067a4b4183944ffb44dfc237: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [461ms] delaying RPC due to: Network error: Failed to connect to peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321): Connection refused: /127.30.240.195:33321, [1636ms] refreshing cache from master, [1636ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [1638ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@1338189423(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@74992161 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086) -> (continuation of Deferred@361723216 after retry RPC@362569229) -> (continuation of Deferred@483920214 after retry RPC@1807958262) -> (continuation of Deferred@431375945 after retry RPC@1753444729), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b3e7270 -> wakeup thread Time-limited test -> (continuation of Deferred@1576780874 after retry RPC@632197239) -> (continuation of Deferred@199291038 after retry RPC@684006086) -> (continuation of Deferred@361723216 after retry RPC@362569229) -> (continuation of Deferred@483920214 after retry RPC@1807958262) -> (continuation of Deferred@431375945 after retry RPC@1753444729))) after lookup
21:04:32.025 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:32.026 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] Successfully connected to peer
21:04:32.046 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x0147ed3a, L:/127.0.0.1:42352 - R:/127.30.240.193:46783] using TOKEN/null
21:04:32.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.049580  1146 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 9 FOLLOWER]: Refusing update from remote peer 6920c90fa92046599c83c12690baf185: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 34. (index mismatch)
21:04:32.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.049630   905 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 -> Peer 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321): Couldn't send request to peer 4764cea6067a4b4183944ffb44dfc237. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.195:33321: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:32.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.050163  1211 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
21:04:32.068 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.195:33321
21:04:32.068 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.069348 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:32.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:33321
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42855
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.156971  1234 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.157119  1234 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.157153  1234 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.160547  1234 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.160638  1234 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.195
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.164031  1234 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.195:33321
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.195
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42855
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1234
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.195
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/logs
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.164810  1234 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.165570  1234 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.170560  1239 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.170694  1234 server_base.cc:1061] running on GCE node
21:04:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.170562  1242 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:32.170622  1240 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.171242  1234 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.171664  1234 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.172842  1234 hybrid_clock.cc:648] HybridClock initialized: now 1783544672172827 us; error 25 us; skew 500 ppm
21:04:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.174459  1234 webserver.cc:534] Webserver started at http://127.30.240.195:42855/ using document root <none> and password file <none>
21:04:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.174980  1234 fs_manager.cc:362] Metadata directory not provided
21:04:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.175052  1234 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.177677  1234 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
21:04:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.179176  1248 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.179901  1234 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.002s	sys 0.000s
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.180008  1234 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.180368  1234 fs_report.cc:389] FS layout report
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/wal
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/data
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.193430  1234 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.193892  1234 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.194031  1234 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.194494  1234 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.195606  1255 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.198134  1234 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.198187  1234 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.198230  1234 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.199536  1234 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.199579  1234 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:04:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.199887  1255 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap starting.
21:04:32.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.208961  1234 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.195:33321
21:04:32.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.209194  1362 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.195:33321 every 8 connection(s)
21:04:32.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.209785  1234 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-2/data/info.pb
21:04:32.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.214962 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 1234
21:04:32.215 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.219969  1364 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.220196  1364 heartbeater.cc:461] Registering TS with master...
21:04:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.220674  1364 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.221132  1365 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.221216  1365 heartbeater.cc:461] Registering TS with master...
21:04:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.221369  1365 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.221570  1255 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Log is configured to *not* fsync() on all Append() calls
21:04:32.222 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.222476 31795 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:32.223 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.193:46783
21:04:32.223 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.223004 31725 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.223788 31725 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.195:55395
21:04:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.224092 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 888
21:04:32.231 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] handling channelInactive
21:04:32.231 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state READY due to: connection closed
21:04:32.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.231825  1363 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.231930  1363 heartbeater.cc:461] Registering TS with master...
21:04:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.232088  1363 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:32.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:32.233 [DEBUG - Time-limited test] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:32.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.233640 31869 ts_manager.cc:194] Re-registered known tserver with Master: 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:32.234 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.234 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.234 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.243 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:32.243 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:32.243 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [10ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@942882449(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@4bc89857 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8080b9f -> wakeup thread Time-limited test)) after lookup
21:04:32.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.243476  1255 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:32.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.243809  1255 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Bootstrap complete.
21:04:32.243 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:32.244 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.244 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.244 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.244828  1255 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent bootstrapping tablet: real 0.045s	user 0.026s	sys 0.011s
21:04:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.246595  1255 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.246848  1255 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Initialized, Role: FOLLOWER
21:04:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.247180  1255 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.247663  1364 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:32.247874  1255 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237: Time spent starting tablet: real 0.003s	user 0.002s	sys 0.001s
21:04:32.263 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:32.263 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:32.263 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [10ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [11ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [12ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [12ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [31ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@942882449(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@4bc89857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8080b9f -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979))) after lookup
21:04:32.264 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:32.265 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.265 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.265 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.283 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:32.283 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:32.283 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [10ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [11ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [12ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [12ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [31ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [32ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [33ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [51ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@942882449(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@4bc89857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8080b9f -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962))) after lookup
21:04:32.284 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:32.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.285 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.343 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:32.343 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:32.343 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=111), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [10ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [11ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [12ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [12ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [31ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [32ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [33ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [51ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [52ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [53ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [53ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [109ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [111ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@942882449(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@4bc89857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962) -> (continuation of Deferred@903502519 after retry RPC@602698309), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8080b9f -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962) -> (continuation of Deferred@903502519 after retry RPC@602698309))) after lookup
21:04:32.344 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:32.344 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.344 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.344 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.743 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:32.743 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)[L]]}]
21:04:32.744 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=511), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [10ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [11ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [12ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [12ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [31ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [32ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [33ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [51ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [52ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [53ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [53ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [109ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [111ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [112ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [112ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [113ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [509ms] refreshing cache from master, [509ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [511ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@942882449(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@4bc89857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962) -> (continuation of Deferred@903502519 after retry RPC@602698309) -> (continuation of Deferred@1058498720 after retry RPC@2133412701), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8080b9f -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962) -> (continuation of Deferred@903502519 after retry RPC@602698309) -> (continuation of Deferred@1058498720 after retry RPC@2133412701))) after lookup
21:04:32.744 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] connecting to peer
21:04:32.745 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.745 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:32.745 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6920c90fa92046599c83c12690baf185(127.30.240.193:46783) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783
21:04:33.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.569336  1376 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 6920c90fa92046599c83c12690baf185)
21:04:33.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.569463  1376 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.570430  1376 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.571189  1079 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.572184  1079 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.575009  1317 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.575217  1317 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 8.
21:04:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.575788  1080 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237; no voters: 6920c90fa92046599c83c12690baf185
21:04:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.575986  1376 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 9 FOLLOWER]: Leader pre-election won for term 10
21:04:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.576067  1376 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 6920c90fa92046599c83c12690baf185)
21:04:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.576107  1376 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 9 FOLLOWER]: Advancing to term 10
21:04:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.576835  1376 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.577178  1376 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 10 election: Requested vote from peers 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.577514  1317 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "42e2bcf086eb4742a41e54464efe56f7" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237"
21:04:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.577636  1317 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 8 FOLLOWER]: Advancing to term 10
21:04:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:33.578068  1079 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111)
21:04:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.579106  1317 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42e2bcf086eb4742a41e54464efe56f7 in term 10.
21:04:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.579401  1080 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237; no voters: 6920c90fa92046599c83c12690baf185
21:04:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.579572  1376 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 10 FOLLOWER]: Leader election won for term 10
21:04:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.579799  1376 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 10 LEADER]: Becoming Leader. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Running, Role: LEADER
21:04:33.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.580065  1376 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:33.582103 31725 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 reported cstate change: term changed from 9 to 10, leader changed from 6920c90fa92046599c83c12690baf185 (127.30.240.193) to 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194). New cstate: current_term: 10 leader_uuid: "42e2bcf086eb4742a41e54464efe56f7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: UNKNOWN } } }
21:04:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.043526  1317 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 10 FOLLOWER]: Refusing update from remote peer 42e2bcf086eb4742a41e54464efe56f7: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
21:04:34.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.044179  1376 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
21:04:34.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.051673  1079 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 -> Peer 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783): Couldn't send request to peer 6920c90fa92046599c83c12690baf185. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.193:46783: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:34.723 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:34.724 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:34.724 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2491), Traces: [0ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [2ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [10ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [11ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [12ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [12ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [31ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [32ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [33ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [48ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [51ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [52ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [53ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [53ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [109ms] refreshing cache from master, [109ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [111ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [112ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [112ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [113ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [509ms] refreshing cache from master, [509ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [511ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [512ms] sending RPC to server 6920c90fa92046599c83c12690baf185, [513ms] received response from server 6920c90fa92046599c83c12690baf185: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [513ms] delaying RPC due to: Network error: Failed to connect to peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783): Connection refused: /127.30.240.193:46783, [2489ms] refreshing cache from master, [2489ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [2491ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@942882449(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@4bc89857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962) -> (continuation of Deferred@903502519 after retry RPC@602698309) -> (continuation of Deferred@1058498720 after retry RPC@2133412701) -> (continuation of Deferred@1402852686 after retry RPC@1725632451), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@8080b9f -> wakeup thread Time-limited test -> (continuation of Deferred@1354035801 after retry RPC@405298979) -> (continuation of Deferred@2113555841 after retry RPC@670159962) -> (continuation of Deferred@903502519 after retry RPC@602698309) -> (continuation of Deferred@1058498720 after retry RPC@2133412701) -> (continuation of Deferred@1402852686 after retry RPC@1725632451))) after lookup
21:04:34.725 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:34.725 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] Successfully connected to peer
21:04:34.737 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x85cd3d03, L:/127.0.0.1:35286 - R:/127.30.240.194:37435] using TOKEN/null
21:04:34.748 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.193:46783
21:04:34.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.749246 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:46783
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37005
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.836447  1412 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.836588  1412 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.836624  1412 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.839358  1412 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.839437  1412 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.193
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.842676  1412 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.193:46783
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.193
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37005
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1412
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.193
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/logs
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.843361  1412 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.844074  1412 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.848837  1417 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.848881  1418 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.848929  1412 server_base.cc:1061] running on GCE node
21:04:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:34.848834  1420 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.849370  1412 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.849798  1412 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.850950  1412 hybrid_clock.cc:648] HybridClock initialized: now 1783544674850935 us; error 24 us; skew 500 ppm
21:04:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.852547  1412 webserver.cc:534] Webserver started at http://127.30.240.193:37005/ using document root <none> and password file <none>
21:04:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.853088  1412 fs_manager.cc:362] Metadata directory not provided
21:04:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.853157  1412 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:34.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.855641  1412 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
21:04:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.857136  1426 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.857873  1412 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.857968  1412 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6920c90fa92046599c83c12690baf185"
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.858275  1412 fs_report.cc:389] FS layout report
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/wal
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/data
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.874888  1412 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.875423  1412 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.875591  1412 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.876017  1412 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.877003  1433 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:34.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.879334  1412 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:34.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.879400  1412 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:34.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.879451  1412 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.880623  1412 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.880678  1412 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
21:04:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.880901  1433 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap starting.
21:04:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.890116  1412 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.193:46783
21:04:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.890193  1540 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.193:46783 every 8 connection(s)
21:04:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.890872  1412 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-0/data/info.pb
21:04:34.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:34.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.894636 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 1412
21:04:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.895426  1433 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Log is configured to *not* fsync() on all Append() calls
21:04:34.901 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:34.902 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.30.240.194:37435
21:04:34.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:34.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.902909 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 1063
21:04:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.905045  1542 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.905215  1542 heartbeater.cc:461] Registering TS with master...
21:04:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.905646  1542 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.907119 31724 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.907997 31724 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.193:60275
21:04:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.910929  1543 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.910974  1541 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.911006  1543 heartbeater.cc:461] Registering TS with master...
21:04:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.911043  1541 heartbeater.cc:461] Registering TS with master...
21:04:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.911141  1543 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.911154  1541 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:34.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.912081 31795 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:34.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.912207 31869 ts_manager.cc:194] Re-registered known tserver with Master: 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:34.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:34.914 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] handling channelInactive
21:04:34.914 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state READY due to: connection closed
21:04:34.914 [DEBUG - Time-limited test] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:34.915 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.915 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.915 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.921062  1433 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:34.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.921371  1433 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Bootstrap complete.
21:04:34.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.922566  1433 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent bootstrapping tablet: real 0.042s	user 0.026s	sys 0.009s
21:04:34.923 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:34.923 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:34.924 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [9ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@436074030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@27e657c5 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736cc55b -> wakeup thread Time-limited test)) after lookup
21:04:34.924 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:34.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.924362  1433 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:34.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.924628  1433 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6920c90fa92046599c83c12690baf185, State: Initialized, Role: FOLLOWER
21:04:34.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.925017  1433 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:34.925 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.925 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.925 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.925760  1433 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185: Time spent starting tablet: real 0.003s	user 0.005s	sys 0.000s
21:04:34.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:34.926227  1542 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:34.942 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:34.942 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:34.942 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [9ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [10ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [11ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [11ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@436074030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@27e657c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736cc55b -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520))) after lookup
21:04:34.943 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:34.943 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.943 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.944 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.983 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:34.983 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:34.983 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [9ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [10ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [11ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [11ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [29ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [30ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [68ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@436074030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@27e657c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736cc55b -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163))) after lookup
21:04:34.983 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:34.984 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.984 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:34.984 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.102 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:35.103 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:35.103 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=189), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [9ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [10ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [11ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [11ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [29ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [30ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [68ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [69ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [70ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [70ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [188ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@436074030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@27e657c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736cc55b -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813))) after lookup
21:04:35.103 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:35.104 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.104 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.104 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.343 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:35.343 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:35.343 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=429), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [9ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [10ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [11ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [11ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [29ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [30ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [68ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [69ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [70ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [70ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [188ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [189ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [190ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [190ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [427ms] refreshing cache from master, [427ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [429ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@436074030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@27e657c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813) -> (continuation of Deferred@1982587544 after retry RPC@988695948), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736cc55b -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813) -> (continuation of Deferred@1982587544 after retry RPC@988695948))) after lookup
21:04:35.344 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:35.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.345 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.463 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:35.463 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)[L],4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321),6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:35.463 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=549), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [9ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [10ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [11ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [11ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [29ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [30ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [68ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [69ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [70ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [70ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [188ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [189ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [190ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [190ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [427ms] refreshing cache from master, [427ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [429ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [429ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [431ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [431ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [547ms] refreshing cache from master, [547ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [549ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@436074030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@27e657c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813) -> (continuation of Deferred@1982587544 after retry RPC@988695948) -> (continuation of Deferred@1370289734 after retry RPC@426132434), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736cc55b -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813) -> (continuation of Deferred@1982587544 after retry RPC@988695948) -> (continuation of Deferred@1370289734 after retry RPC@426132434))) after lookup
21:04:35.464 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] connecting to peer
21:04:35.464 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.465 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] cleaning up while in state CONNECTING due to: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:35.465 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435) for tablet 8a5460f709204d5fac0a735d09169599: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435
21:04:36.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.229329  1554 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:36.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.229511  1554 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:36.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.230697  1554 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195:33321)
21:04:36.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.231314  1428 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:36.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.232065  1428 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:36.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.234300  1317 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "6920c90fa92046599c83c12690baf185" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "4764cea6067a4b4183944ffb44dfc237" is_pre_election: true
21:04:36.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.234726  1429 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7, 4764cea6067a4b4183944ffb44dfc237
21:04:36.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.234954  1554 raft_consensus.cc:2749] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
21:04:36.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.248589  1559 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:36.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.248665  1559 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:36.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.249341  1559 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:36.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.249889  1250 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:36.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.250731  1250 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:36.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.253391  1495 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185" is_pre_election: true
21:04:36.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.253602  1495 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 9.
21:04:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.254182  1250 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7
21:04:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.254347  1559 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 10 FOLLOWER]: Leader pre-election won for term 11
21:04:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.254412  1559 raft_consensus.cc:493] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 42e2bcf086eb4742a41e54464efe56f7)
21:04:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.254451  1559 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 10 FOLLOWER]: Advancing to term 11
21:04:36.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.255134  1559 raft_consensus.cc:515] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:36.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.255432  1559 leader_election.cc:290] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 11 election: Requested vote from peers 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435), 6920c90fa92046599c83c12690baf185 (127.30.240.193:46783)
21:04:36.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.255887  1495 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8a5460f709204d5fac0a735d09169599" candidate_uuid: "4764cea6067a4b4183944ffb44dfc237" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "6920c90fa92046599c83c12690baf185"
21:04:36.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.255975  1495 raft_consensus.cc:3060] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 9 FOLLOWER]: Advancing to term 11
21:04:36.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.256273  1250 leader_election.cc:336] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111)
21:04:36.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.257331  1495 raft_consensus.cc:2468] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4764cea6067a4b4183944ffb44dfc237 in term 11.
21:04:36.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.257581  1250 leader_election.cc:304] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4764cea6067a4b4183944ffb44dfc237, 6920c90fa92046599c83c12690baf185; no voters: 42e2bcf086eb4742a41e54464efe56f7
21:04:36.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.257711  1559 raft_consensus.cc:2804] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 11 FOLLOWER]: Leader election won for term 11
21:04:36.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.257884  1559 raft_consensus.cc:697] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [term 11 LEADER]: Becoming Leader. State: Replica: 4764cea6067a4b4183944ffb44dfc237, State: Running, Role: LEADER
21:04:36.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.258191  1559 consensus_queue.cc:237] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:36.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.260200 31724 catalog_manager.cc:5696] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 reported cstate change: term changed from 10 to 11, leader changed from 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194) to 4764cea6067a4b4183944ffb44dfc237 (127.30.240.195). New cstate: current_term: 11 leader_uuid: "4764cea6067a4b4183944ffb44dfc237" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } health_report { overall_health: HEALTHY } } }
21:04:36.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.694015  1495 raft_consensus.cc:1275] T 8a5460f709204d5fac0a735d09169599 P 6920c90fa92046599c83c12690baf185 [term 11 FOLLOWER]: Refusing update from remote peer 4764cea6067a4b4183944ffb44dfc237: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
21:04:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:36.694638  1559 consensus_queue.cc:1048] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
21:04:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:36.702227  1250 consensus_peers.cc:597] T 8a5460f709204d5fac0a735d09169599 P 4764cea6067a4b4183944ffb44dfc237 -> Peer 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435): Couldn't send request to peer 42e2bcf086eb4742a41e54464efe56f7. Status: Network error: Client connection negotiation failed: client connection to 127.30.240.194:37435: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:38.383 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8a5460f709204d5fac0a735d09169599 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126' with partition [<start>, <end>)
21:04:38.384 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=8a5460f709204d5fac0a735d09169599@[42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435),4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)[L],6920c90fa92046599c83c12690baf185(127.30.240.193:46783)]}]
21:04:38.384 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=8a5460f709204d5fac0a735d09169599, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=3470), Traces: [0ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [1ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [1ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [7ms] refreshing cache from master, [7ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [9ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [10ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [11ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [11ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [28ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [29ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [29ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [30ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [68ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [69ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [70ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [70ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [187ms] refreshing cache from master, [187ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [188ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [189ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [190ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [190ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [427ms] refreshing cache from master, [427ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [429ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [429ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [431ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [431ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [547ms] refreshing cache from master, [547ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [549ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, [549ms] sending RPC to server 42e2bcf086eb4742a41e54464efe56f7, [551ms] received response from server 42e2bcf086eb4742a41e54464efe56f7: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [551ms] delaying RPC due to: Network error: Failed to connect to peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435): Connection refused: /127.30.240.194:37435, [3467ms] refreshing cache from master, [3467ms] Sub RPC GetTableLocations: sending RPC to server master-127.30.240.254:45497, [3469ms] Sub RPC GetTableLocations: received response from server master-127.30.240.254:45497: OK, deferred=Deferred@436074030(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f6ff436bed0@27e657c5 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813) -> (continuation of Deferred@1982587544 after retry RPC@988695948) -> (continuation of Deferred@1370289734 after retry RPC@426132434) -> (continuation of Deferred@2018468330 after retry RPC@1402783202), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@736cc55b -> wakeup thread Time-limited test -> (continuation of Deferred@267478416 after retry RPC@446836520) -> (continuation of Deferred@1045995311 after retry RPC@889728163) -> (continuation of Deferred@1261203407 after retry RPC@1068842813) -> (continuation of Deferred@1982587544 after retry RPC@988695948) -> (continuation of Deferred@1370289734 after retry RPC@426132434) -> (continuation of Deferred@2018468330 after retry RPC@1402783202))) after lookup
21:04:38.385 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] connecting to peer
21:04:38.386 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] Successfully connected to peer
21:04:38.400 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x8737c226, L:/127.0.0.1:36230 - R:/127.30.240.195:33321] using TOKEN/null
21:04:38.413 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.30.240.194:37435
21:04:38.413 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.413832 31683 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:04:38.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.500793  1596 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:04:38.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.500981  1596 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:38.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.501051  1596 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:04:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.504099  1596 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:04:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.504230  1596 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.240.194
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.507272  1596 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.240.212:36757
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.240.194:37435
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.240.194
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34591
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.240.254:45497,127.30.240.253:42945,127.30.240.252:45579
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1596
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.240.194
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/logs
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:46 UTC on bdcb31816ec0
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16875
21:04:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.508251  1596 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:38.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.509063  1596 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.514281  1604 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.514277  1601 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:04:38.514375  1602 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.514469  1596 server_base.cc:1061] running on GCE node
21:04:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.514693  1596 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:38.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.515110  1596 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:38.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.516237  1596 hybrid_clock.cc:648] HybridClock initialized: now 1783544678516224 us; error 26 us; skew 500 ppm
21:04:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.517553  1596 webserver.cc:534] Webserver started at http://127.30.240.194:34591/ using document root <none> and password file <none>
21:04:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.517930  1596 fs_manager.cc:362] Metadata directory not provided
21:04:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.517987  1596 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:38.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.520418  1596 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
21:04:38.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.521883  1610 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:04:38.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.522825  1596 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.522904  1596 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data,/tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42e2bcf086eb4742a41e54464efe56f7"
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:04:10 on dist-test-slave-dr23"
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.523200  1596 fs_report.cc:389] FS layout report
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/wal
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/data
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:38.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.532783  1596 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:04:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.533198  1596 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.533325  1596 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.533712  1596 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:04:38.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.534580  1617 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:04:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.537212  1596 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:04:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.537289  1596 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
21:04:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.537345  1596 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:04:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.538502  1596 ts_tablet_manager.cc:616] Registered 1 tablets
21:04:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.538543  1596 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:04:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.538851  1617 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap starting.
21:04:38.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.547727  1596 rpc_server.cc:307] RPC server started. Bound to: 127.30.240.194:37435
21:04:38.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.547811  1724 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.240.194:37435 every 8 connection(s)
21:04:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.548617  1596 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQ9WW30/test-tmp/mini-kudu-cluster5376199577211520457/ts-1/data/info.pb
21:04:38.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.554607  1617 log.cc:826] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Log is configured to *not* fsync() on all Append() calls
21:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.557376  1727 heartbeater.cc:344] Connected to a master server at 127.30.240.253:42945
21:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.557391  1726 heartbeater.cc:344] Connected to a master server at 127.30.240.254:45497
21:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.557437  1725 heartbeater.cc:344] Connected to a master server at 127.30.240.252:45579
21:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.557569  1727 heartbeater.cc:461] Registering TS with master...
21:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.557569  1725 heartbeater.cc:461] Registering TS with master...
21:04:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.558039  1725 heartbeater.cc:507] Master 127.30.240.252:45579 requested a full tablet report, sending...
21:04:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.558039  1727 heartbeater.cc:507] Master 127.30.240.253:42945 requested a full tablet report, sending...
21:04:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.558631 31683 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu as pid 1596
21:04:38.558 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:04:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.557590  1726 heartbeater.cc:461] Registering TS with master...
21:04:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.559240  1726 heartbeater.cc:507] Master 127.30.240.254:45497 requested a full tablet report, sending...
21:04:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.559741 31795 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.559768 31869 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.560308 31725 ts_manager.cc:194] Re-registered known tserver with Master: 42e2bcf086eb4742a41e54464efe56f7 (127.30.240.194:37435)
21:04:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.561115 31725 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.240.194:58469
21:04:38.566 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:38.570 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544651126, tablet=8a5460f709204d5fac0a735d09169599, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:04:38.575 [DEBUG - main] (Connection.java:575) [peer master-127.30.240.253:42945(127.30.240.253:42945)] disconnecting while in state READY
21:04:38.576 [DEBUG - main] (Connection.java:575) [peer 42e2bcf086eb4742a41e54464efe56f7(127.30.240.194:37435)] disconnecting while in state TERMINATED
21:04:38.576 [DEBUG - main] (Connection.java:575) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] disconnecting while in state READY
21:04:38.576 [DEBUG - main] (Connection.java:575) [peer master-127.30.240.254:45497(127.30.240.254:45497)] disconnecting while in state READY
21:04:38.577 [DEBUG - main] (Connection.java:575) [peer master-127.30.240.252:45579(127.30.240.252:45579)] disconnecting while in state READY
21:04:38.578 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.240.252:45579(127.30.240.252:45579)] handling channelInactive
21:04:38.578 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.30.240.253:42945(127.30.240.253:42945)] handling channelInactive
21:04:38.578 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.30.240.253:42945(127.30.240.253:42945)] cleaning up while in state READY due to: connection closed
21:04:38.579 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.240.254:45497(127.30.240.254:45497)] handling channelInactive
21:04:38.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.240.254:45497(127.30.240.254:45497)] cleaning up while in state READY due to: connection closed
21:04:38.579 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] handling channelInactive
21:04:38.579 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 4764cea6067a4b4183944ffb44dfc237(127.30.240.195:33321)] cleaning up while in state READY due to: connection closed
21:04:38.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.240.252:45579(127.30.240.252:45579)] cleaning up while in state READY due to: connection closed
21:04:38.580 [DEBUG - main] (Connection.java:575) [peer 6920c90fa92046599c83c12690baf185(127.30.240.193:46783)] disconnecting while in state TERMINATED
21:04:38.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.580544  1617 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:04:38.580 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:04:38.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.580881  1617 tablet_bootstrap.cc:492] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Bootstrap complete.
21:04:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.582013  1617 ts_tablet_manager.cc:1403] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent bootstrapping tablet: real 0.043s	user 0.035s	sys 0.004s
21:04:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.582537 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 1412
21:04:38.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.585661  1617 raft_consensus.cc:359] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:38.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.585919  1617 raft_consensus.cc:740] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42e2bcf086eb4742a41e54464efe56f7, State: Initialized, Role: FOLLOWER
21:04:38.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.586261  1617 consensus_queue.cc:260] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42e2bcf086eb4742a41e54464efe56f7" member_type: VOTER last_known_addr { host: "127.30.240.194" port: 37435 } } peers { permanent_uuid: "6920c90fa92046599c83c12690baf185" member_type: VOTER last_known_addr { host: "127.30.240.193" port: 46783 } } peers { permanent_uuid: "4764cea6067a4b4183944ffb44dfc237" member_type: VOTER last_known_addr { host: "127.30.240.195" port: 33321 } }
21:04:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.586750  1726 heartbeater.cc:499] Master 127.30.240.254:45497 was elected leader, sending a full tablet report...
21:04:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.586959  1617 ts_tablet_manager.cc:1434] T 8a5460f709204d5fac0a735d09169599 P 42e2bcf086eb4742a41e54464efe56f7: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
21:04:38.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.591459 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 1596
21:04:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.597927 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 1234
21:04:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.603083 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 31694
21:04:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.608563 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 31763
21:04:38.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:04:38.613781 31683 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQ9WW30/build/debug/bin/kudu with pid 31835
21:04:38.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:04:38Z chronyd exiting

Time: 29.709

OK (1 test)

