JUnit version 4.13.2
.00:13:23.315 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1786493603307
00:13:23.376 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
00:13:23.412 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:13:23.426 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskC8xxxa/test-tmp
00:13:23.458 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
00:13:23.462 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskC8xxxa/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
00:13:23.670 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

00:13:23.690 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:13:23.692 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

00:13:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.694121 29060 mini_oidc.cc:150] Starting JWKS server
00:13:23.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.696341 29060 webserver.cc:533] Webserver started at http://127.0.0.1:43469/ using document root <none> and password file <none>
00:13:23.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.696511 29060 mini_oidc.cc:186] Starting OIDC Discovery server
00:13:23.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.696791 29060 webserver.cc:533] Webserver started at http://127.0.0.1:40881/ using document root <none> and password file <none>
00:13:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:13:23Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
00:13:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:13:23Z Disabled control of system clock
00:13:23.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.715093 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:23.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:23.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/wal
00:13:23.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data
00:13:23.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:23.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:23.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/logs
00:13:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data/info.pb
00:13:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:23.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:23.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:23.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:23.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:23.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:23.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.62:40027
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.62
00:13:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:13:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:13:23.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43469/jwks.json with env {}
00:13:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.830112 29082 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.830358 29082 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.830405 29082 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.834409 29082 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:13:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.834491 29082 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.834511 29082 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:13:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.834530 29082 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:13:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.834548 29082 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:13:23.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.834563 29082 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43469/jwks.json
00:13:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.839042 29082 master_runner.cc:387] Master server non-default flags:
00:13:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data
00:13:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/wal
00:13:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:13:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.62:40027
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43469/jwks.json
00:13:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data/info.pb
00:13:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.62
00:13:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29082
00:13:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/logs
00:13:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:13:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.840611 29082 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.842002 29082 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.848026 29095 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:23.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.848028 29089 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:23.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.848301 29098 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:23.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.849112 29082 server_base.cc:1061] running on GCE node
00:13:23.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.849766 29082 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:23.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.850898 29082 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.852119 29082 hybrid_clock.cc:648] HybridClock initialized: now 1786493603851830 us; error 302 us; skew 500 ppm
00:13:23.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.854166 29082 webserver.cc:533] Webserver started at http://127.28.97.62:36579/ using document root <none> and password file <none>
00:13:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.854853 29082 fs_manager.cc:362] Metadata directory not provided
00:13:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.854946 29082 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.855180 29082 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:13:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.856956 29082 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data/instance:
00:13:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9cfa1d5ebfe6476c952402a90304b9f9"
00:13:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:23 on dist-test-slave-3h5h"
00:13:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.857508 29082 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/wal/instance:
00:13:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9cfa1d5ebfe6476c952402a90304b9f9"
00:13:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:23 on dist-test-slave-3h5h"
00:13:23.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.861279 29082 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.004s	sys 0.000s
00:13:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.863927 29105 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.865209 29082 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
00:13:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.865362 29082 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/wal
00:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9cfa1d5ebfe6476c952402a90304b9f9"
00:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:23 on dist-test-slave-3h5h"
00:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.865482 29082 fs_report.cc:389] FS layout report
00:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/wal
00:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/wal
00:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data/data
00:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:23.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.907956 29082 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:23.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.908932 29082 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:23.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.909158 29082 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:23.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.917614 29082 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.62:40027
00:13:23.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.917649 29191 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.62:40027 every 8 connection(s)
00:13:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.918918 29082 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/data/info.pb
00:13:23.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.922129 29192 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:13:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.923256 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 29082
00:13:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.923419 29060 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-0/wal/instance
00:13:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.925201 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/wal
00:13:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data
00:13:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/logs
00:13:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data/info.pb
00:13:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:13:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:13:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:13:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.61:39769
00:13:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.61
00:13:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:13:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:13:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43469/jwks.json with env {}
00:13:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.926237 29192 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } has no permanent_uuid. Determining permanent_uuid...
00:13:23.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.943512 29192 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } has no permanent_uuid. Determining permanent_uuid...
00:13:23.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.944523 29111 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.61:39769: connect: Connection refused (error 111)
00:13:23.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.945770 29192 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.61:39769: Network error: Client connection negotiation failed: client connection to 127.28.97.61:39769: connect: Connection refused (error 111)
00:13:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:23.995095 29192 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } attempt: 1
00:13:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:23.996871 29192 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.61:39769: Network error: Client connection negotiation failed: client connection to 127.28.97.61:39769: connect: Connection refused (error 111)
00:13:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.039809 29196 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.040081 29196 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.040132 29196 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.043979 29196 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:13:24.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.044064 29196 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.044086 29196 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:13:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.044107 29196 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:13:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.044126 29196 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:13:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.044142 29196 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43469/jwks.json
00:13:24.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.049619 29196 master_runner.cc:387] Master server non-default flags:
00:13:24.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data
00:13:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/wal
00:13:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:13:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:13:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.61:39769
00:13:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:13:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:13:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43469/jwks.json
00:13:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data/info.pb
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.61
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29196
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/logs
00:13:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:13:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:24.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.050770 29196 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:24.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.052220 29196 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.058961 29208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.059067 29196 server_base.cc:1061] running on GCE node
00:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.059192 29209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.058979 29211 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.061146 29196 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.062572 29196 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:24.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.063786 29196 hybrid_clock.cc:648] HybridClock initialized: now 1786493604063759 us; error 54 us; skew 500 ppm
00:13:24.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.065088 29192 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } attempt: 2
00:13:24.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.066443 29196 webserver.cc:533] Webserver started at http://127.28.97.61:44885/ using document root <none> and password file <none>
00:13:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.066708 29192 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.61:39769: Network error: Client connection negotiation failed: client connection to 127.28.97.61:39769: connect: Connection refused (error 111)
00:13:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.067211 29196 fs_manager.cc:362] Metadata directory not provided
00:13:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.067296 29196 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.067538 29196 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:13:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.069656 29196 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data/instance:
00:13:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef32351099bc4a86800d3fe14d4390bb"
00:13:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.070116 29196 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/wal/instance:
00:13:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef32351099bc4a86800d3fe14d4390bb"
00:13:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.074779 29196 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.005s	sys 0.000s
00:13:24.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.077384 29222 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.078500 29196 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
00:13:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.078604 29196 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/wal
00:13:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef32351099bc4a86800d3fe14d4390bb"
00:13:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.078681 29196 fs_report.cc:389] FS layout report
00:13:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/wal
00:13:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/wal
00:13:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data/data
00:13:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.090610 29196 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.091347 29196 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.091535 29196 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.101769 29294 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.61:39769 every 8 connection(s)
00:13:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.101766 29196 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.61:39769
00:13:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.102962 29196 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/data/info.pb
00:13:24.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.104183 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 29196
00:13:24.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.104287 29060 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-1/wal/instance
00:13:24.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.105386 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/wal
00:13:24.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data
00:13:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/logs
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data/info.pb
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:13:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:13:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:13:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.60:44689
00:13:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.60
00:13:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:13:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:13:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43469/jwks.json with env {}
00:13:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.106670 29296 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:13:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.110030 29296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } has no permanent_uuid. Determining permanent_uuid...
00:13:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.122119 29296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } has no permanent_uuid. Determining permanent_uuid...
00:13:24.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.129323 29296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } has no permanent_uuid. Determining permanent_uuid...
00:13:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.130302 29226 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.60:44689: connect: Connection refused (error 111)
00:13:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.131615 29296 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.60:44689: Network error: Client connection negotiation failed: client connection to 127.28.97.60:44689: connect: Connection refused (error 111)
00:13:24.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.157872 29192 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } attempt: 3
00:13:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.162979 29192 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } has no permanent_uuid. Determining permanent_uuid...
00:13:24.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.164561 29192 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.60:44689: Network error: Client connection negotiation failed: client connection to 127.28.97.60:44689: connect: Connection refused (error 111)
00:13:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.180925 29296 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } attempt: 1
00:13:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.182551 29296 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.60:44689: Network error: Client connection negotiation failed: client connection to 127.28.97.60:44689: connect: Connection refused (error 111)
00:13:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.195767 29192 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } attempt: 1
00:13:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.197707 29192 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.60:44689: Network error: Client connection negotiation failed: client connection to 127.28.97.60:44689: connect: Connection refused (error 111)
00:13:24.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.226696 29300 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:24.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.226922 29300 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:24.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.226969 29300 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:24.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.230664 29300 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:13:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.230739 29300 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.230760 29300 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:13:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.230778 29300 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:13:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.230798 29300 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:13:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.230813 29300 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43469/jwks.json
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.234879 29300 master_runner.cc:387] Master server non-default flags:
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/wal
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.60:44689
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43469/jwks.json
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data/info.pb
00:13:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.60
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29300
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/logs
00:13:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:13:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.236004 29300 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.237437 29300 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.245033 29310 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.245033 29312 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.245149 29300 server_base.cc:1061] running on GCE node
00:13:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.245038 29316 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.245995 29300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.247459 29300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.250793 29296 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } attempt: 2
00:13:24.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.252755 29296 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.60:44689: Network error: Client connection negotiation failed: client connection to 127.28.97.60:44689: connect: Connection refused (error 111)
00:13:24.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.254930 29300 hybrid_clock.cc:648] HybridClock initialized: now 1786493604254866 us; error 84 us; skew 500 ppm
00:13:24.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.257218 29300 webserver.cc:533] Webserver started at http://127.28.97.60:33637/ using document root <none> and password file <none>
00:13:24.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.257920 29300 fs_manager.cc:362] Metadata directory not provided
00:13:24.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.258042 29300 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:24.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.258248 29300 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:13:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.259953 29300 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data/instance:
00:13:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a629b20b4a34a37bed224a341a97b75"
00:13:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.260397 29300 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/wal/instance:
00:13:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a629b20b4a34a37bed224a341a97b75"
00:13:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.263818 29300 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.006s	sys 0.000s
00:13:24.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.266398 29324 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.267923 29300 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
00:13:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.268085 29300 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/wal
00:13:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a629b20b4a34a37bed224a341a97b75"
00:13:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.268199 29300 fs_report.cc:389] FS layout report
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/wal
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/wal
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data/data
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:24.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.272892 29192 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } attempt: 2
00:13:24.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.274529 29192 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.97.60:44689: Network error: Client connection negotiation failed: client connection to 127.28.97.60:44689: connect: Connection refused (error 111)
00:13:24.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.288656 29300 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:24.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.289574 29300 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.289803 29300 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:24.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.297900 29300 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.60:44689
00:13:24.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.297945 29413 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.60:44689 every 8 connection(s)
00:13:24.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.299168 29300 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/data/info.pb
00:13:24.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.300402 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 29300
00:13:24.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.300518 29060 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/master-2/wal/instance
00:13:24.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.303176 29414 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:13:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.308028 29414 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } has no permanent_uuid. Determining permanent_uuid...
00:13:24.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.319339 29414 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } has no permanent_uuid. Determining permanent_uuid...
00:13:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.324018 29414 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } has no permanent_uuid. Determining permanent_uuid...
00:13:24.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.330200 29414 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75: Bootstrap starting.
00:13:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.332659 29414 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75: Neither blocks nor log segments found. Creating new log.
00:13:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.333705 29414 log.cc:826] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75: Log is configured to *not* fsync() on all Append() calls
00:13:24.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.336519 29414 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75: No bootstrap required, opened a new log
00:13:24.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.339641 29414 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.339814 29414 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:13:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.339850 29414 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a629b20b4a34a37bed224a341a97b75, State: Initialized, Role: FOLLOWER
00:13:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.340274 29414 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.341130 29424 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.341259 29424 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: This master's current role is: FOLLOWER
00:13:24.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.341741 29414 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: configured and running, proceeding with master startup.
00:13:24.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.343943 29296 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } attempt: 3
00:13:24.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.347621 29441 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:13:24.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.348039 29441 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:13:24.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.345507 29300 master_runner.cc:187] Error getting master registration for 127.28.97.62:40027: OK, instance_id { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" instance_seqno: 1786493603915425 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:13:24.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.348481 29300 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:13:24.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.350612 29296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: Bootstrap starting.
00:13:24.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.353638 29296 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: Neither blocks nor log segments found. Creating new log.
00:13:24.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.354674 29296 log.cc:826] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: Log is configured to *not* fsync() on all Append() calls
00:13:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.356786 29296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: No bootstrap required, opened a new log
00:13:24.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.361238 29296 raft_consensus.cc:359] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.361470 29296 raft_consensus.cc:385] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:13:24.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.361541 29296 raft_consensus.cc:740] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef32351099bc4a86800d3fe14d4390bb, State: Initialized, Role: FOLLOWER
00:13:24.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.362051 29296 consensus_queue.cc:260] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.362600 29447 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.362737 29447 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [sys.catalog]: This master's current role is: FOLLOWER
00:13:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.363488 29296 sys_catalog.cc:565] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [sys.catalog]: configured and running, proceeding with master startup.
00:13:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.367413 29424 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:13:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.367599 29424 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.368673 29424 leader_election.cc:290] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9cfa1d5ebfe6476c952402a90304b9f9 (127.28.97.62:40027), ef32351099bc4a86800d3fe14d4390bb (127.28.97.61:39769)
00:13:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.369231 29464 catalog_manager.cc:1594] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:13:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.369305 29464 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:13:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.367833 29196 master_runner.cc:187] Error getting master registration for 127.28.97.62:40027: OK, instance_id { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" instance_seqno: 1786493603915425 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:13:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.370580 29196 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:13:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.371630 29264 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a629b20b4a34a37bed224a341a97b75" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef32351099bc4a86800d3fe14d4390bb" is_pre_election: true
00:13:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.371891 29264 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3a629b20b4a34a37bed224a341a97b75 in term 0.
00:13:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.372360 29331 leader_election.cc:304] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3a629b20b4a34a37bed224a341a97b75, ef32351099bc4a86800d3fe14d4390bb; no voters: 
00:13:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.372648 29424 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Leader pre-election won for term 1
00:13:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.372745 29424 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:13:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.372794 29424 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 0 FOLLOWER]: Advancing to term 1
00:13:24.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.373708 29192 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } attempt: 3
00:13:24.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.374143 29424 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.374647 29424 leader_election.cc:290] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [CANDIDATE]: Term 1 election: Requested vote from peers 9cfa1d5ebfe6476c952402a90304b9f9 (127.28.97.62:40027), ef32351099bc4a86800d3fe14d4390bb (127.28.97.61:39769)
00:13:24.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.375476 29264 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a629b20b4a34a37bed224a341a97b75" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef32351099bc4a86800d3fe14d4390bb"
00:13:24.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.375594 29264 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [term 0 FOLLOWER]: Advancing to term 1
00:13:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.376784 29264 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3a629b20b4a34a37bed224a341a97b75 in term 1.
00:13:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.377367 29331 leader_election.cc:304] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3a629b20b4a34a37bed224a341a97b75, ef32351099bc4a86800d3fe14d4390bb; no voters: 
00:13:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.377738 29424 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 1 FOLLOWER]: Leader election won for term 1
00:13:24.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.378094 29424 raft_consensus.cc:697] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [term 1 LEADER]: Becoming Leader. State: Replica: 3a629b20b4a34a37bed224a341a97b75, State: Running, Role: LEADER
00:13:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.378518 29424 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.380564 29424 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a629b20b4a34a37bed224a341a97b75. Latest consensus state: current_term: 1 leader_uuid: "3a629b20b4a34a37bed224a341a97b75" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.380666 29424 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: This master's current role is: LEADER
00:13:24.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.387971 29192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: Bootstrap starting.
00:13:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.389151 29468 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
00:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.390561 29468 catalog_manager.cc:1520] Initializing Kudu cluster ID...
00:13:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.390931 29192 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: Neither blocks nor log segments found. Creating new log.
00:13:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.392436 29192 log.cc:826] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: Log is configured to *not* fsync() on all Append() calls
00:13:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.400257 29264 raft_consensus.cc:1275] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [term 1 FOLLOWER]: Refusing update from remote peer 3a629b20b4a34a37bed224a341a97b75: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:13:24.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.401041 29424 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:13:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.404376 29192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: No bootstrap required, opened a new log
00:13:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.406955 29447 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a629b20b4a34a37bed224a341a97b75. Latest consensus state: current_term: 1 leader_uuid: "3a629b20b4a34a37bed224a341a97b75" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.407104 29447 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [sys.catalog]: This master's current role is: FOLLOWER
00:13:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.408185 29192 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.408703 29192 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:13:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.409016 29192 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9cfa1d5ebfe6476c952402a90304b9f9, State: Initialized, Role: FOLLOWER
00:13:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.409819 29192 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } }
00:13:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.411484 29192 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [sys.catalog]: configured and running, proceeding with master startup.
00:13:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.411767 29137 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [term 0 FOLLOWER]: Advancing to term 1
00:13:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.411870 29141 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a629b20b4a34a37bed224a341a97b75" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" is_pre_election: true
00:13:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.412343 29138 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a629b20b4a34a37bed224a341a97b75" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9cfa1d5ebfe6476c952402a90304b9f9"
00:13:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.413180 29137 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [term 1 FOLLOWER]: Refusing update from remote peer 3a629b20b4a34a37bed224a341a97b75: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:13:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.416436 29466 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:13:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.416591 29475 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.416699 29475 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [sys.catalog]: This master's current role is: FOLLOWER
00:13:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.421265 29424 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3a629b20b4a34a37bed224a341a97b75" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.421402 29424 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: This master's current role is: LEADER
00:13:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.421736 29447 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3a629b20b4a34a37bed224a341a97b75" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.421840 29447 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb [sys.catalog]: This master's current role is: FOLLOWER
00:13:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.422391 29473 mvcc.cc:204] Tried to move back new op lower bound from 7317477803616477184 to 7317477803538309120. Current Snapshot: MvccSnapshot[applied={T|T < 7317477803616477184}]
00:13:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.422806 29491 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:13:24.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.423249 29491 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:13:24.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.424280 29475 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a629b20b4a34a37bed224a341a97b75. Latest consensus state: current_term: 1 leader_uuid: "3a629b20b4a34a37bed224a341a97b75" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.424394 29475 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [sys.catalog]: This master's current role is: FOLLOWER
00:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.426765 29468 catalog_manager.cc:1383] Generated new cluster ID: 6ab7b0ea85de4cdeb14f8fdcb85f9af5
00:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.426836 29468 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
00:13:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.428776 29466 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3a629b20b4a34a37bed224a341a97b75" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.428928 29466 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75 [sys.catalog]: This master's current role is: LEADER
00:13:24.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.431860 29475 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3a629b20b4a34a37bed224a341a97b75" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9cfa1d5ebfe6476c952402a90304b9f9" member_type: VOTER last_known_addr { host: "127.28.97.62" port: 40027 } } peers { permanent_uuid: "ef32351099bc4a86800d3fe14d4390bb" member_type: VOTER last_known_addr { host: "127.28.97.61" port: 39769 } } peers { permanent_uuid: "3a629b20b4a34a37bed224a341a97b75" member_type: VOTER last_known_addr { host: "127.28.97.60" port: 44689 } } }
00:13:24.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.431982 29475 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9 [sys.catalog]: This master's current role is: FOLLOWER
00:13:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.432353 29498 mvcc.cc:204] Tried to move back new op lower bound from 7317477803616477184 to 7317477803538309120. Current Snapshot: MvccSnapshot[applied={T|T < 7317477803616477184}]
00:13:24.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.459750 29468 catalog_manager.cc:1406] Generated new certificate authority record
00:13:24.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.461546 29468 catalog_manager.cc:1540] Loading token signing keys...
00:13:24.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.489204 29468 catalog_manager.cc:6092] T 00000000000000000000000000000000 P 3a629b20b4a34a37bed224a341a97b75: Generated new TSK 0
00:13:24.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.490571 29468 catalog_manager.cc:1550] Initializing in-progress tserver states...
00:13:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.510227 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:0
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.682956 29502 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.683265 29502 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.683331 29502 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:24.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.688750 29502 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:24.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.688941 29502 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.1
00:13:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.695396 29502 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:0
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29502
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.697124 29502 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.699023 29502 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.707543 29521 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.707512 29517 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.707705 29518 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.709487 29502 server_base.cc:1061] running on GCE node
00:13:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.709931 29502 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.710685 29502 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.711886 29502 hybrid_clock.cc:648] HybridClock initialized: now 1786493604711827 us; error 75 us; skew 500 ppm
00:13:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.714408 29502 webserver.cc:533] Webserver started at http://127.28.97.1:39491/ using document root <none> and password file <none>
00:13:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.715134 29502 fs_manager.cc:362] Metadata directory not provided
00:13:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.715214 29502 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.715463 29502 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:13:24.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.717597 29502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/instance:
00:13:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.718096 29502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal/instance:
00:13:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.722259 29502 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.000s	sys 0.004s
00:13:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.725137 29530 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.726447 29502 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.726581 29502 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.726697 29502 fs_report.cc:389] FS layout report
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/data
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.749706 29502 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.750456 29502 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.750617 29502 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.751151 29502 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.752279 29502 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:13:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.752331 29502 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:13:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.752370 29502 ts_tablet_manager.cc:616] Registered 0 tablets
00:13:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.752385 29502 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:13:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.762732 29502 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.1:41751
00:13:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.762768 29711 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.1:41751 every 8 connection(s)
00:13:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.763751 29502 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.772444 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 29502
00:13:24.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.772576 29060 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal/instance
00:13:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.774063 29712 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.774354 29716 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.774093 29713 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.774597 29713 heartbeater.cc:461] Registering TS with master...
00:13:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.774595 29712 heartbeater.cc:461] Registering TS with master...
00:13:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.774596 29716 heartbeater.cc:461] Registering TS with master...
00:13:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.775266 29713 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.775285 29716 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:24.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.776727 29127 ts_manager.cc:194] Registered new tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.776798 29247 ts_manager.cc:194] Registered new tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.778187 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:0
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.778311 29712 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.779609 29356 ts_manager.cc:194] Registered new tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.781088 29356 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.1:50111
00:13:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.892516 29722 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.892758 29722 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.892802 29722 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.896436 29722 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.896564 29722 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.2
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.900761 29722 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:0
00:13:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29722
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.901993 29722 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.903100 29722 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:24.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.910322 29727 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.910398 29722 server_base.cc:1061] running on GCE node
00:13:24.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.910409 29734 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:24.910322 29730 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:24.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.911003 29722 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:24.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.911651 29722 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.913106 29722 hybrid_clock.cc:648] HybridClock initialized: now 1786493604913058 us; error 69 us; skew 500 ppm
00:13:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.915055 29722 webserver.cc:533] Webserver started at http://127.28.97.2:44183/ using document root <none> and password file <none>
00:13:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.915582 29722 fs_manager.cc:362] Metadata directory not provided
00:13:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.915665 29722 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.915932 29722 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:13:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.917675 29722 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/instance:
00:13:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.918273 29722 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal/instance:
00:13:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.922497 29722 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.001s	sys 0.004s
00:13:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.925223 29741 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.926479 29722 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.003s
00:13:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.926590 29722 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.926683 29722 fs_report.cc:389] FS layout report
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/data
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:24.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.959422 29722 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:24.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.960114 29722 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:24.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.960268 29722 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.960822 29722 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.962077 29722 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:13:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.962129 29722 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:13:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.962172 29722 ts_tablet_manager.cc:616] Registered 0 tablets
00:13:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.962188 29722 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:13:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.972090 29722 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.2:35557
00:13:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.972148 29927 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.2:35557 every 8 connection(s)
00:13:24.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.973855 29722 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.975401 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 29722
00:13:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.975509 29060 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal/instance
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.978271 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:0
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.985183 29930 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.985209 29931 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.985260 29928 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.985474 29928 heartbeater.cc:461] Registering TS with master...
00:13:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.985474 29930 heartbeater.cc:461] Registering TS with master...
00:13:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.985476 29931 heartbeater.cc:461] Registering TS with master...
00:13:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.986156 29928 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.986311 29930 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.986560 29931 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.987650 29127 ts_manager.cc:194] Registered new tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.987679 29356 ts_manager.cc:194] Registered new tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.987869 29247 ts_manager.cc:194] Registered new tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:24.988384 29356 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.2:36045
00:13:25.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.095178 29941 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:25.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.095413 29941 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:25.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.095456 29941 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.099033 29941 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.099153 29941 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.3
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.103932 29941 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:0
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29941
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.105314 29941 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.106659 29941 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.115484 29948 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.115486 29950 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:25.115486 29947 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.116737 29941 server_base.cc:1061] running on GCE node
00:13:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.117228 29941 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:25.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.117939 29941 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:25.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.119125 29941 hybrid_clock.cc:648] HybridClock initialized: now 1786493605119093 us; error 49 us; skew 500 ppm
00:13:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.121177 29941 webserver.cc:533] Webserver started at http://127.28.97.3:38861/ using document root <none> and password file <none>
00:13:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.121824 29941 fs_manager.cc:362] Metadata directory not provided
00:13:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.121913 29941 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.122184 29941 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:13:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.123869 29941 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/instance:
00:13:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:25 on dist-test-slave-3h5h"
00:13:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.124442 29941 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal/instance:
00:13:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:25 on dist-test-slave-3h5h"
00:13:25.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.128001 29941 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.006s	sys 0.000s
00:13:25.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.130558 29964 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:25.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.131671 29941 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
00:13:25.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.131807 29941 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:25 on dist-test-slave-3h5h"
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.131907 29941 fs_report.cc:389] FS layout report
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/data
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.148084 29941 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:25.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.148985 29941 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:25.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.149210 29941 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:25.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.149928 29941 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.151153 29941 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:13:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.151226 29941 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:13:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.151284 29941 ts_tablet_manager.cc:616] Registered 0 tablets
00:13:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.151309 29941 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:13:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.161348 29941 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.3:44737
00:13:25.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.162014 30151 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.3:44737 every 8 connection(s)
00:13:25.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.162817 29941 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:25.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.169902 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 29941
00:13:25.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.170023 29060 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal/instance
00:13:25.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.173452 30154 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:25.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.173499 30152 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.173769 30154 heartbeater.cc:461] Registering TS with master...
00:13:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.173769 30152 heartbeater.cc:461] Registering TS with master...
00:13:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.174445 30154 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.174448 30152 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.174957 30155 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.175042 30155 heartbeater.cc:461] Registering TS with master...
00:13:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.175225 30155 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.175619 29356 ts_manager.cc:194] Registered new tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.175706 29127 ts_manager.cc:194] Registered new tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.175927 29247 ts_manager.cc:194] Registered new tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.176332 29356 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.3:34993
00:13:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.179564 29060 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
00:13:25.180 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:13:25.181 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

00:13:25.190 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.28.97.62"
      port: 40027
    }
    bound_http_address {
      host: "127.28.97.62"
      port: 36579
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.28.97.61"
      port: 39769
    }
    bound_http_address {
      host: "127.28.97.61"
      port: 44885
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.28.97.60"
      port: 44689
    }
    bound_http_address {
      host: "127.28.97.60"
      port: 33637
    }
  }
}

00:13:25.194 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

00:13:25.196 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.28.97.1"
      port: 41751
    }
    bound_http_address {
      host: "127.28.97.1"
      port: 39491
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.28.97.2"
      port: 35557
    }
    bound_http_address {
      host: "127.28.97.2"
      port: 44183
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.28.97.3"
      port: 44737
    }
    bound_http_address {
      host: "127.28.97.3"
      port: 38861
    }
  }
}

00:13:25.196 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:13:25.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.370997 29464 catalog_manager.cc:1295] Loaded cluster ID: 6ab7b0ea85de4cdeb14f8fdcb85f9af5
00:13:25.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.371101 29464 catalog_manager.cc:1588] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: loading cluster ID for follower catalog manager: success
00:13:25.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.374595 29464 catalog_manager.cc:1610] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: acquiring CA information for follower catalog manager: success
00:13:25.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.376389 29464 catalog_manager.cc:1638] T 00000000000000000000000000000000 P ef32351099bc4a86800d3fe14d4390bb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:13:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.425195 29491 catalog_manager.cc:1295] Loaded cluster ID: 6ab7b0ea85de4cdeb14f8fdcb85f9af5
00:13:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.425298 29491 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: loading cluster ID for follower catalog manager: success
00:13:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.428290 29491 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: acquiring CA information for follower catalog manager: success
00:13:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.429908 29491 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 9cfa1d5ebfe6476c952402a90304b9f9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:13:25.445 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
00:13:25.520 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
00:13:25.697 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.97.61:39769(127.28.97.61:39769)] connecting to peer
00:13:25.753 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.97.62:40027(127.28.97.62:40027)] connecting to peer
00:13:25.755 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.97.60:44689(127.28.97.60:44689)] connecting to peer
00:13:25.776 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.97.61:39769(127.28.97.61:39769)] Successfully connected to peer
00:13:25.776 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.28.97.60:44689(127.28.97.60:44689)] Successfully connected to peer
00:13:25.776 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.97.62:40027(127.28.97.62:40027)] Successfully connected to peer
00:13:25.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.783289 29712 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:25.876 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.28.97.60
00:13:25.876 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.28.97.62
00:13:25.876 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.28.97.61
00:13:25.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:25.990612 29928 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:26.156 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xbad77db3, L:/127.0.0.1:33472 - R:/127.28.97.62:40027] using SASL/PLAIN
00:13:26.156 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x5631882b, L:/127.0.0.1:52556 - R:/127.28.97.60:44689] using SASL/PLAIN
00:13:26.157 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xdad0e803, L:/127.0.0.1:37106 - R:/127.28.97.61:39769] using SASL/PLAIN
00:13:26.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.178581 30152 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:26.205 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
00:13:26.206 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=549), Traces: [0ms] refreshing cache from master, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.97.61:39769, [93ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.97.62:40027, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.97.60:44689, [514ms] Sub RPC ConnectToMaster: received response from server master-127.28.97.62:40027: OK, [514ms] Sub RPC ConnectToMaster: received response from server master-127.28.97.61:39769: OK, [515ms] Sub RPC ConnectToMaster: received response from server master-127.28.97.60:44689: OK, deferred=null) after lookup
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.257814 29353 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52556:
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642"
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
00:13:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:13:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:13:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:13:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:26.259392 29353 catalog_manager.cc:7099] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
00:13:26.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.294946 29824 tablet_service.cc:1511] Processing CreateTablet for tablet 577804064edd41e4802dc27206597d97 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642 [id=722c3a11425544f7ab63637baac23626]), partition=RANGE (key) PARTITION UNBOUNDED
00:13:26.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.296883 29824 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 577804064edd41e4802dc27206597d97. 1 dirs total, 0 dirs full, 0 dirs failed
00:13:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.297433 30031 tablet_service.cc:1511] Processing CreateTablet for tablet 577804064edd41e4802dc27206597d97 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642 [id=722c3a11425544f7ab63637baac23626]), partition=RANGE (key) PARTITION UNBOUNDED
00:13:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.300653 29604 tablet_service.cc:1511] Processing CreateTablet for tablet 577804064edd41e4802dc27206597d97 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642 [id=722c3a11425544f7ab63637baac23626]), partition=RANGE (key) PARTITION UNBOUNDED
00:13:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.301967 29604 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 577804064edd41e4802dc27206597d97. 1 dirs total, 0 dirs full, 0 dirs failed
00:13:26.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.305218 30031 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 577804064edd41e4802dc27206597d97. 1 dirs total, 0 dirs full, 0 dirs failed
00:13:26.315 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 722c3a11425544f7ab63637baac23626
00:13:26.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.316466 30196 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap starting.
00:13:26.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.318238 30197 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap starting.
00:13:26.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.320647 30196 tablet_bootstrap.cc:654] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Neither blocks nor log segments found. Creating new log.
00:13:26.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.320647 30197 tablet_bootstrap.cc:654] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Neither blocks nor log segments found. Creating new log.
00:13:26.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.321738 30196 log.cc:826] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Log is configured to *not* fsync() on all Append() calls
00:13:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.321776 30197 log.cc:826] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Log is configured to *not* fsync() on all Append() calls
00:13:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.322697 30194 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap starting.
00:13:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.324185 30196 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: No bootstrap required, opened a new log
00:13:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.324185 30197 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: No bootstrap required, opened a new log
00:13:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.324417 30197 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent bootstrapping tablet: real 0.006s	user 0.004s	sys 0.000s
00:13:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.324417 30196 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent bootstrapping tablet: real 0.009s	user 0.005s	sys 0.000s
00:13:26.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.325168 30194 tablet_bootstrap.cc:654] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Neither blocks nor log segments found. Creating new log.
00:13:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.326370 30194 log.cc:826] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Log is configured to *not* fsync() on all Append() calls
00:13:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.328439 30197 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.328439 30196 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.328613 30194 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: No bootstrap required, opened a new log
00:13:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.328685 30196 raft_consensus.cc:385] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:13:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.328744 30196 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f962932df48b42568f69e73e7e4fd476, State: Initialized, Role: FOLLOWER
00:13:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.328878 30194 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent bootstrapping tablet: real 0.006s	user 0.006s	sys 0.000s
00:13:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.329303 30197 raft_consensus.cc:385] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:13:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.329344 30196 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.329609 30197 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Initialized, Role: FOLLOWER
00:13:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.330200 30197 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.330380 30196 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent starting tablet: real 0.006s	user 0.007s	sys 0.000s
00:13:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.331307 30197 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent starting tablet: real 0.007s	user 0.006s	sys 0.000s
00:13:26.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.333225 30194 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.333691 30194 raft_consensus.cc:385] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:13:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.333985 30194 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Initialized, Role: FOLLOWER
00:13:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.334673 30194 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.335760 30194 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent starting tablet: real 0.007s	user 0.005s	sys 0.000s
00:13:26.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:26.415446 30157 tablet.cc:2406] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:13:26.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.439297 30210 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:13:26.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.439540 30210 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.441116 30210 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:26.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.446476 30066 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" is_pre_election: true
00:13:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.446831 29633 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476" is_pre_election: true
00:13:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.446866 30066 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 0.
00:13:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.447106 29633 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 0.
00:13:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.447530 29744 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a; no voters: 
00:13:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.448019 30210 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Leader pre-election won for term 1
00:13:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.448312 30210 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:13:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.448442 30210 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 0 FOLLOWER]: Advancing to term 1
00:13:26.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.449618 30210 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.450174 30210 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 1 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:26.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.450706 30066 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:26.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.450863 30066 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 0 FOLLOWER]: Advancing to term 1
00:13:26.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.451009 29633 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:26.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.451152 29633 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 0 FOLLOWER]: Advancing to term 1
00:13:26.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.452260 30066 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 1.
00:13:26.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.452512 29633 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 1.
00:13:26.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.453372 29744 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476; no voters: 
00:13:26.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.453688 30210 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 1 FOLLOWER]: Leader election won for term 1
00:13:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.454042 30210 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 1 LEADER]: Becoming Leader. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Running, Role: LEADER
00:13:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.454627 30210 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.458657 29356 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a reported cstate change: term changed from 0 to 1, leader changed from <none> to e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2). New cstate: current_term: 1 leader_uuid: "e9fbd638be254004a6d0df9c49d25e3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:26.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:26.476233 29933 tablet.cc:2406] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:13:26.506 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 722c3a11425544f7ab63637baac23626
00:13:26.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:26.516431 29717 tablet.cc:2406] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:13:26.529 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:26.537 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:26.539 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [9ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=null) after lookup
00:13:26.558 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:26.562 [DEBUG - kudu-nio-3] (Connection.java:757) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] Successfully connected to peer
00:13:26.648 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x117e7e88, L:/127.0.0.1:52762 - R:/127.28.97.2:35557] using TOKEN/null
00:13:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.655891 29633 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Refusing update from remote peer e9fbd638be254004a6d0df9c49d25e3a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:13:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.655891 30066 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Refusing update from remote peer e9fbd638be254004a6d0df9c49d25e3a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:13:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.656639 30210 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:13:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.657135 30219 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:13:26.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.666946 30223 mvcc.cc:204] Tried to move back new op lower bound from 7317477812854599680 to 7317477812042207232. Current Snapshot: MvccSnapshot[applied={T|T < 7317477812854599680}]
00:13:26.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.670049 30222 mvcc.cc:204] Tried to move back new op lower bound from 7317477812854599680 to 7317477812042207232. Current Snapshot: MvccSnapshot[applied={T|T < 7317477812854599680}]
00:13:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.678267 30224 mvcc.cc:204] Tried to move back new op lower bound from 7317477812854599680 to 7317477812042207232. Current Snapshot: MvccSnapshot[applied={T|T < 7317477812854599680}]
00:13:26.780 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:26.782 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.2:35557
00:13:26.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:26.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:26.784557 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 29722
00:13:26.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:26.816 [DEBUG - kudu-nio-3] (Connection.java:239) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] handling channelInactive
00:13:26.817 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state READY due to: connection closed
00:13:26.817 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: connection closed
00:13:26.841 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:26.842 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:26.842 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=28), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [3ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [27ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1520110905(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@15d50c8c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7e711160 -> wakeup thread Time-limited test)) after lookup
00:13:26.843 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:26.846 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.846 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.846 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.861 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:26.861 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:26.862 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=48), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [3ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [27ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [29ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [32ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [47ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1520110905(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@15d50c8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7e711160 -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693))) after lookup
00:13:26.863 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:26.865 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.866 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.866 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.902 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:26.902 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:26.903 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=89), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [3ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [27ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [29ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [32ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [47ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [48ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [52ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [52ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1520110905(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@15d50c8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7e711160 -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448))) after lookup
00:13:26.904 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:26.906 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.907 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.907 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.981 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:26.982 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:26.982 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=168), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [3ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [27ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [29ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [32ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [47ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [48ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [52ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [52ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [90ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [93ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [93ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [167ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1520110905(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@15d50c8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7e711160 -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208))) after lookup
00:13:26.983 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:26.984 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.985 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:26.985 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:27.381 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:27.381 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:27.382 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=568), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [3ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [27ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [29ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [32ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [47ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [48ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [52ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [52ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [90ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [93ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [93ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [167ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [168ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [171ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [171ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [563ms] refreshing cache from master, [563ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [566ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1520110905(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@15d50c8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208) -> (continuation of Deferred@429772801 after retry RPC@269462252), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7e711160 -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208) -> (continuation of Deferred@429772801 after retry RPC@269462252))) after lookup
00:13:27.383 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:27.384 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:27.384 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:27.384 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:28.121 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:28.122 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:28.122 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1308), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [3ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [27ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [29ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [32ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [47ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [48ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [52ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [52ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [90ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [93ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [93ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [167ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [168ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [171ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [171ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [563ms] refreshing cache from master, [563ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [566ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [568ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [570ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [570ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [1303ms] refreshing cache from master, [1303ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [1307ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1520110905(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@15d50c8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208) -> (continuation of Deferred@429772801 after retry RPC@269462252) -> (continuation of Deferred@181494624 after retry RPC@1743052828), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7e711160 -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208) -> (continuation of Deferred@429772801 after retry RPC@269462252) -> (continuation of Deferred@181494624 after retry RPC@1743052828))) after lookup
00:13:28.123 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:28.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:28.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:28.125 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:28.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.211704 30246 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:28.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.211776 30248 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:28.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.211900 30246 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:28.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.211892 30248 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:28.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.213404 30248 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:28.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.213479 30246 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:28.214807 29531 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:28.214815 29965 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:28.216116 29531 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:28.216555 29965 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.218429 30066 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" is_pre_election: true
00:13:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.218559 29633 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476" is_pre_election: true
00:13:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.218601 30066 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f962932df48b42568f69e73e7e4fd476 in term 1.
00:13:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.218686 29633 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 1.
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219087 29969 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219086 29532 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219336 30248 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Leader pre-election won for term 2
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219390 30248 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Starting leader election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219432 30248 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 1 FOLLOWER]: Advancing to term 2
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219548 30246 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Leader pre-election won for term 2
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219619 30246 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Starting leader election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.219666 30246 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 1 FOLLOWER]: Advancing to term 2
00:13:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.220352 30248 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.220605 30246 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.220783 30248 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 2 election: Requested vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.221071 30246 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 2 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.221591 29633 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.221629 30066 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.221774 29633 raft_consensus.cc:2393] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 8f51a58ed1ee47e1ba1b670c49abba2b in current term 2: Already voted for candidate f962932df48b42568f69e73e7e4fd476 in this term.
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.221807 30066 raft_consensus.cc:2393] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate f962932df48b42568f69e73e7e4fd476 in current term 2: Already voted for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in this term.
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:28.222395 29531 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:28.222396 29965 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.222465 29531 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.222517 29965 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b; no voters: e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.222635 30246 raft_consensus.cc:2749] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:28.222774 30248 raft_consensus.cc:2749] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
00:13:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.901007 30262 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:13:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.901157 30262 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.901706 30262 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.902313 29633 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476" is_pre_election: true
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.902443 29633 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 2.
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.902837 29969 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: 
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.903059 30262 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Leader pre-election won for term 3
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.903128 30262 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.903187 30262 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 2 FOLLOWER]: Advancing to term 3
00:13:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:29.903327 29965 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:29.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.905891 30262 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:29.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.906338 30262 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 3 election: Requested vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.906885 29633 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.907058 29633 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 2 FOLLOWER]: Advancing to term 3
00:13:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:29.907384 29965 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.908034 29633 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 3.
00:13:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.908414 29969 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.908623 30262 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 3 FOLLOWER]: Leader election won for term 3
00:13:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.908804 30262 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 3 LEADER]: Becoming Leader. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Running, Role: LEADER
00:13:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.909416 30262 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:29.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:29.912187 29356 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b reported cstate change: term changed from 1 to 3, leader changed from e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2) to 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3). New cstate: current_term: 3 leader_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:30.101 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:30.102 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:30.103 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3288), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [3ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [27ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [29ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [32ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [47ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [48ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [52ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [52ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [83ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [90ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [93ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [93ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [164ms] refreshing cache from master, [164ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [167ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [168ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [171ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [171ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [563ms] refreshing cache from master, [563ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [566ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [568ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [570ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [570ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [1303ms] refreshing cache from master, [1303ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [1307ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [1309ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [1311ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [1311ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [3284ms] refreshing cache from master, [3284ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [3287ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1520110905(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@15d50c8c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208) -> (continuation of Deferred@429772801 after retry RPC@269462252) -> (continuation of Deferred@181494624 after retry RPC@1743052828) -> (continuation of Deferred@51598776 after retry RPC@1127723462), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7e711160 -> wakeup thread Time-limited test -> (continuation of Deferred@770882839 after retry RPC@1547431693) -> (continuation of Deferred@1015486802 after retry RPC@1144640448) -> (continuation of Deferred@1646810156 after retry RPC@1382968208) -> (continuation of Deferred@429772801 after retry RPC@269462252) -> (continuation of Deferred@181494624 after retry RPC@1743052828) -> (continuation of Deferred@51598776 after retry RPC@1127723462))) after lookup
00:13:30.104 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:30.105 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] Successfully connected to peer
00:13:30.131 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x7df6d369, L:/127.0.0.1:46888 - R:/127.28.97.3:44737] using TOKEN/null
00:13:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.135492 29633 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 3 FOLLOWER]: Refusing update from remote peer 8f51a58ed1ee47e1ba1b670c49abba2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
00:13:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.136059 30266 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
00:13:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.136332 29965 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b -> Peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Couldn't send request to peer e9fbd638be254004a6d0df9c49d25e3a. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:30.158 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.2:35557
00:13:30.160 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.161082 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.277463 30295 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.277700 30295 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.277745 30295 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:30.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.281131 30295 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:30.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.281240 30295 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.2
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.285364 30295 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30295
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.286511 30295 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.287611 30295 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.295228 30304 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.295233 30303 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:30.295334 30309 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.295662 30295 server_base.cc:1061] running on GCE node
00:13:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.296069 30295 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.296741 30295 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:30.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.297889 30295 hybrid_clock.cc:648] HybridClock initialized: now 1786493610297869 us; error 34 us; skew 500 ppm
00:13:30.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.300041 30295 webserver.cc:533] Webserver started at http://127.28.97.2:44183/ using document root <none> and password file <none>
00:13:30.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.300707 30295 fs_manager.cc:362] Metadata directory not provided
00:13:30.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.300794 30295 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.304932 30295 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
00:13:30.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.307349 30316 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.308537 30295 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.308641 30295 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.309088 30295 fs_report.cc:389] FS layout report
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/data
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.331270 30295 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.332005 30295 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.332172 30295 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.332719 30295 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:30.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.334153 30332 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.337792 30295 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.337877 30295 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.001s
00:13:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.337946 30295 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.339495 30295 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.339565 30295 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
00:13:30.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.339922 30332 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap starting.
00:13:30.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.355369 30295 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.2:35557
00:13:30.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.355543 30505 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.2:35557 every 8 connection(s)
00:13:30.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.357004 30295 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:30.360 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.359723 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 30295
00:13:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.362017 30332 log.cc:826] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Log is configured to *not* fsync() on all Append() calls
00:13:30.371 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:30.371 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.3:44737
00:13:30.372 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:13:30.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.372736 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 29941
00:13:30.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.375607 30332 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:30.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.376145 30332 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap complete.
00:13:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.377446 30332 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent bootstrapping tablet: real 0.038s	user 0.024s	sys 0.007s
00:13:30.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.379833 30332 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:30.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.380211 30332 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Initialized, Role: FOLLOWER
00:13:30.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.380664 30332 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:30.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.381946 30332 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent starting tablet: real 0.004s	user 0.000s	sys 0.003s
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.383517 30507 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.383862 30507 heartbeater.cc:461] Registering TS with master...
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.384606 30507 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.385730 30510 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.385831 30510 heartbeater.cc:461] Registering TS with master...
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.386108 30510 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.386400 30513 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.386487 30513 heartbeater.cc:461] Registering TS with master...
00:13:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.386713 30513 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.387029 29356 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.387899 29248 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.387985 29356 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.2:53423
00:13:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.389252 29127 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:30.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:30.390120 30507 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:30.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:30.396 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] handling channelInactive
00:13:30.396 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state READY due to: connection closed
00:13:30.397 [DEBUG - Time-limited test] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:30.398 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.398 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.398 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.422 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:30.422 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:30.423 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@235125411(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@38866f3e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4922de51 -> wakeup thread Time-limited test)) after lookup
00:13:30.423 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:30.424 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.424 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.425 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.441 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:30.442 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:30.442 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [29ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [29ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@235125411(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@38866f3e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4922de51 -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123))) after lookup
00:13:30.443 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:30.444 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.445 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.445 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.481 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:30.482 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:30.482 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=86), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [29ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [29ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [49ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [49ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@235125411(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@38866f3e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4922de51 -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540))) after lookup
00:13:30.483 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:30.484 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.484 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.484 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.621 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:30.622 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:30.622 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=226), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [29ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [29ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [49ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [49ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [86ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [88ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [88ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [221ms] refreshing cache from master, [222ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [225ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@235125411(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@38866f3e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540) -> (continuation of Deferred@1636688179 after retry RPC@1139037549), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4922de51 -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540) -> (continuation of Deferred@1636688179 after retry RPC@1139037549))) after lookup
00:13:30.623 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:30.625 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.625 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.625 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.702 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:30.702 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:30.703 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=306), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [29ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [29ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [49ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [49ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [86ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [88ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [88ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [221ms] refreshing cache from master, [222ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [225ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [227ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [229ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [229ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [302ms] refreshing cache from master, [302ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [305ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@235125411(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@38866f3e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540) -> (continuation of Deferred@1636688179 after retry RPC@1139037549) -> (continuation of Deferred@1532503761 after retry RPC@390434139), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4922de51 -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540) -> (continuation of Deferred@1636688179 after retry RPC@1139037549) -> (continuation of Deferred@1532503761 after retry RPC@390434139))) after lookup
00:13:30.704 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:30.706 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:30.706 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.657768 30528 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 8f51a58ed1ee47e1ba1b670c49abba2b)
00:13:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.657934 30528 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.659081 30528 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:31.660233 29532 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:31.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.664546 30429 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "e9fbd638be254004a6d0df9c49d25e3a" is_pre_election: true
00:13:31.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.664955 30429 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f962932df48b42568f69e73e7e4fd476 in term 1.
00:13:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.666090 29531 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b
00:13:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.666345 30528 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 3 FOLLOWER]: Leader pre-election won for term 4
00:13:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.666419 30528 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 8f51a58ed1ee47e1ba1b670c49abba2b)
00:13:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.666453 30528 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 3 FOLLOWER]: Advancing to term 4
00:13:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.667327 30528 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.667758 30528 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 4 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.668304 30429 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.668504 30429 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 1 FOLLOWER]: Advancing to term 4
00:13:31.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:31.669358 29532 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.670694 30429 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f962932df48b42568f69e73e7e4fd476 in term 4.
00:13:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.671123 29531 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b
00:13:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.671392 30528 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 4 FOLLOWER]: Leader election won for term 4
00:13:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.671622 30528 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 4 LEADER]: Becoming Leader. State: Replica: f962932df48b42568f69e73e7e4fd476, State: Running, Role: LEADER
00:13:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.672132 30528 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:31.675086 29356 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 reported cstate change: term changed from 3 to 4, leader changed from 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3) to f962932df48b42568f69e73e7e4fd476 (127.28.97.1). New cstate: current_term: 4 leader_uuid: "f962932df48b42568f69e73e7e4fd476" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: HEALTHY } } }
00:13:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.057484 30429 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 4 FOLLOWER]: Refusing update from remote peer f962932df48b42568f69e73e7e4fd476: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
00:13:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.058295 30528 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
00:13:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.067865 29532 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 -> Peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Couldn't send request to peer 8f51a58ed1ee47e1ba1b670c49abba2b. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:32.121 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:32.122 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:32.122 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1726), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [29ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [29ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [49ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [49ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [86ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [88ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [88ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [221ms] refreshing cache from master, [222ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [225ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [227ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [229ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [229ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [302ms] refreshing cache from master, [302ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [305ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [308ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [310ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [310ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1721ms] refreshing cache from master, [1722ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [1725ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@235125411(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@38866f3e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540) -> (continuation of Deferred@1636688179 after retry RPC@1139037549) -> (continuation of Deferred@1532503761 after retry RPC@390434139) -> (continuation of Deferred@521326053 after retry RPC@612807689), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4922de51 -> wakeup thread Time-limited test -> (continuation of Deferred@1565667686 after retry RPC@48082123) -> (continuation of Deferred@419728696 after retry RPC@1045664540) -> (continuation of Deferred@1636688179 after retry RPC@1139037549) -> (continuation of Deferred@1532503761 after retry RPC@390434139) -> (continuation of Deferred@521326053 after retry RPC@612807689))) after lookup
00:13:32.123 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:32.125 [DEBUG - kudu-nio-1] (Connection.java:757) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] Successfully connected to peer
00:13:32.146 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x2fe57fee, L:/127.0.0.1:36336 - R:/127.28.97.1:41751] using TOKEN/null
00:13:32.169 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.3:44737
00:13:32.170 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.171322 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:44737
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38861
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:32.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.293673 30574 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:32.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.293888 30574 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:32.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.293936 30574 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:32.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.297529 30574 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:32.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.297641 30574 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.3
00:13:32.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.302752 30574 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:44737
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38861
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30574
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.304071 30574 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.305576 30574 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.312947 30587 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.312947 30584 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:32.313012 30590 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.313460 30574 server_base.cc:1061] running on GCE node
00:13:32.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.313822 30574 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:32.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.314396 30574 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:32.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.315585 30574 hybrid_clock.cc:648] HybridClock initialized: now 1786493612315565 us; error 35 us; skew 500 ppm
00:13:32.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.317621 30574 webserver.cc:533] Webserver started at http://127.28.97.3:38861/ using document root <none> and password file <none>
00:13:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.318274 30574 fs_manager.cc:362] Metadata directory not provided
00:13:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.318385 30574 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:32.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.322144 30574 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.004s	sys 0.000s
00:13:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.324295 30600 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:32.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.325651 30574 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.325800 30574 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:25 on dist-test-slave-3h5h"
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.326371 30574 fs_report.cc:389] FS layout report
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/data
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.352279 30574 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:32.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.353199 30574 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:32.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.353430 30574 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:32.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.354233 30574 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:32.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.356067 30611 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.359974 30574 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.360065 30574 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
00:13:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.360133 30574 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:32.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.362185 30574 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:32.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.362246 30574 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
00:13:32.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.362681 30611 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap starting.
00:13:32.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.375828 30574 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.3:44737
00:13:32.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.375860 30786 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.3:44737 every 8 connection(s)
00:13:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.377018 30574 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.379319 30611 log.cc:826] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Log is configured to *not* fsync() on all Append() calls
00:13:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.381505 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 30574
00:13:32.381 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.388500 30788 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.388616 30792 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.389034 30792 heartbeater.cc:461] Registering TS with master...
00:13:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.389057 30788 heartbeater.cc:461] Registering TS with master...
00:13:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.389681 30792 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.389679 30788 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.391134 30787 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.391222 30787 heartbeater.cc:461] Registering TS with master...
00:13:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.391476 30787 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.391784 29248 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:32.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.392129 29127 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:32.392 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.392644 29356 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:32.393 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.1:41751
00:13:32.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:13:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.393734 29356 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.3:42819
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.394312 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 29502
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.415130 30611 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.415640 30611 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap complete.
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.417032 30611 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent bootstrapping tablet: real 0.055s	user 0.016s	sys 0.016s
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.420549 30611 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.421072 30611 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Initialized, Role: FOLLOWER
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.421651 30611 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.423038 30611 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
00:13:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:32.427635 30787 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:32.436 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:32.436 [DEBUG - kudu-nio-1] (Connection.java:239) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] handling channelInactive
00:13:32.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state READY due to: connection closed
00:13:32.438 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:32.439 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.439 [DEBUG - kudu-nio-2] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.440 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.462 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:32.462 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:32.462 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=3, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [1ms] delaying RPC due to: Illegal state: connection is terminated, [2ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1898861512(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@7abba074 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c357cf2 -> wakeup thread Time-limited test)) after lookup
00:13:32.463 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:32.464 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.464 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.465 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.481 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:32.481 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:32.482 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=5, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [1ms] delaying RPC due to: Illegal state: connection is terminated, [2ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [29ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [29ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1898861512(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@7abba074 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c357cf2 -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104))) after lookup
00:13:32.482 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:32.483 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.483 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.484 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.521 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:32.522 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:32.522 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=7, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=85), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [1ms] delaying RPC due to: Illegal state: connection is terminated, [2ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [29ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [29ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [46ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [48ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [48ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1898861512(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@7abba074 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c357cf2 -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095))) after lookup
00:13:32.523 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:32.524 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.524 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.524 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.581 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:32.581 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:32.582 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=9, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=145), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [1ms] delaying RPC due to: Illegal state: connection is terminated, [2ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [29ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [29ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [46ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [48ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [48ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [86ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [88ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [88ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [145ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1898861512(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@7abba074 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095) -> (continuation of Deferred@46282931 after retry RPC@292274342), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c357cf2 -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095) -> (continuation of Deferred@46282931 after retry RPC@292274342))) after lookup
00:13:32.582 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:32.584 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.584 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:32.584 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:33.022 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:33.022 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:33.022 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=11, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=585), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [1ms] delaying RPC due to: Illegal state: connection is terminated, [2ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [29ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [29ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [46ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [48ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [48ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [86ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [88ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [88ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [145ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [146ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [148ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [148ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [582ms] refreshing cache from master, [582ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [585ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1898861512(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@7abba074 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095) -> (continuation of Deferred@46282931 after retry RPC@292274342) -> (continuation of Deferred@1740355450 after retry RPC@1171118715), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c357cf2 -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095) -> (continuation of Deferred@46282931 after retry RPC@292274342) -> (continuation of Deferred@1740355450 after retry RPC@1171118715))) after lookup
00:13:33.023 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:33.025 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:33.025 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:33.025 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:33.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.668793 30810 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 4 FOLLOWER]: Starting pre-election (detected failure of leader f962932df48b42568f69e73e7e4fd476)
00:13:33.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.669072 30810 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:33.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.670274 30810 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:33.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:33.671159 30328 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:33.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:33.672246 30328 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.675645 30706 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" is_pre_election: true
00:13:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.675909 30706 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 3.
00:13:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.676599 30322 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a; no voters: f962932df48b42568f69e73e7e4fd476
00:13:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.676882 30810 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 4 FOLLOWER]: Leader pre-election won for term 5
00:13:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.676975 30810 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 4 FOLLOWER]: Starting leader election (detected failure of leader f962932df48b42568f69e73e7e4fd476)
00:13:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.677022 30810 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 4 FOLLOWER]: Advancing to term 5
00:13:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.677937 30810 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.678387 30810 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 5 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.678879 30706 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.679172 30706 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 3 FOLLOWER]: Advancing to term 5
00:13:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:33.679787 30328 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.681288 30706 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 5.
00:13:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.681618 30322 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a; no voters: f962932df48b42568f69e73e7e4fd476
00:13:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.681792 30810 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 5 FOLLOWER]: Leader election won for term 5
00:13:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.682065 30810 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 5 LEADER]: Becoming Leader. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Running, Role: LEADER
00:13:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.682404 30810 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:33.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:33.685207 29356 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a reported cstate change: term changed from 4 to 5, leader changed from f962932df48b42568f69e73e7e4fd476 (127.28.97.1) to e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2). New cstate: current_term: 5 leader_uuid: "e9fbd638be254004a6d0df9c49d25e3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:34.229700 30328 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a -> Peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Couldn't send request to peer f962932df48b42568f69e73e7e4fd476. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:34.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:34.298545 30706 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 5 FOLLOWER]: Refusing update from remote peer e9fbd638be254004a6d0df9c49d25e3a: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
00:13:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:34.301756 30833 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
00:13:34.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:34.309289 30838 mvcc.cc:204] Tried to move back new op lower bound from 7317477835425878016 to 7317477833412407296. Current Snapshot: MvccSnapshot[applied={T|T < 7317477835364048896}]
00:13:35.762 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:35.762 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:35.762 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=13, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3326), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [1ms] delaying RPC due to: Illegal state: connection is terminated, [2ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [25ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [27ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [29ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [29ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [45ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [46ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [48ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [48ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [85ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [86ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [88ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [88ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [145ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [146ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [148ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [148ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [582ms] refreshing cache from master, [582ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [585ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [587ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [589ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [589ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [3322ms] refreshing cache from master, [3322ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [3325ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1898861512(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@7abba074 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095) -> (continuation of Deferred@46282931 after retry RPC@292274342) -> (continuation of Deferred@1740355450 after retry RPC@1171118715) -> (continuation of Deferred@1238768941 after retry RPC@1659045998), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c357cf2 -> wakeup thread Time-limited test -> (continuation of Deferred@1909676004 after retry RPC@108423104) -> (continuation of Deferred@1930927066 after retry RPC@1189784095) -> (continuation of Deferred@46282931 after retry RPC@292274342) -> (continuation of Deferred@1740355450 after retry RPC@1171118715) -> (continuation of Deferred@1238768941 after retry RPC@1659045998))) after lookup
00:13:35.763 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:35.764 [DEBUG - kudu-nio-0] (Connection.java:757) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] Successfully connected to peer
00:13:35.785 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x9e7e64b5, L:/127.0.0.1:49860 - R:/127.28.97.2:35557] using TOKEN/null
00:13:35.807 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.1:41751
00:13:35.807 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.808409 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:41751
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.918421 30869 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.918659 30869 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.918744 30869 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:35.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.922318 30869 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:35.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.922434 30869 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.1
00:13:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.926497 30869 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:41751
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30869
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:35.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.927830 30869 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.929533 30869 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.936878 30881 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.936928 30876 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:35.936955 30877 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.937609 30869 server_base.cc:1061] running on GCE node
00:13:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.938050 30869 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.938673 30869 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:35.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.939857 30869 hybrid_clock.cc:648] HybridClock initialized: now 1786493615939834 us; error 36 us; skew 500 ppm
00:13:35.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.941793 30869 webserver.cc:533] Webserver started at http://127.28.97.1:39491/ using document root <none> and password file <none>
00:13:35.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.942405 30869 fs_manager.cc:362] Metadata directory not provided
00:13:35.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.942509 30869 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.946249 30869 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.002s
00:13:35.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.948463 30891 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:35.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.949560 30869 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.949698 30869 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.950187 30869 fs_report.cc:389] FS layout report
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/data
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:35.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:35.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.970613 30869 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:35.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.971377 30869 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:35.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.971599 30869 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.972288 30869 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:35.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.973825 30905 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.977583 30869 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.977685 30869 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
00:13:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.977756 30869 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.979346 30869 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.979418 30869 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
00:13:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.979760 30905 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap starting.
00:13:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.991237 30869 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.1:41751
00:13:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.991310 31083 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.1:41751 every 8 connection(s)
00:13:35.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.992452 30869 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:35.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.995309 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 30869
00:13:35.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:36.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:35.999135 30905 log.cc:826] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Log is configured to *not* fsync() on all Append() calls
00:13:36.007 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:36.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.013614 31084 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:36.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.014585 31088 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:36.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.014698 31085 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:36.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.014868 31085 heartbeater.cc:461] Registering TS with master...
00:13:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.014720 31088 heartbeater.cc:461] Registering TS with master...
00:13:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.014720 31084 heartbeater.cc:461] Registering TS with master...
00:13:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.015651 31088 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.015651 31084 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.015717 31085 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.018002 29247 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.018827 29353 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.019943 29353 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.1:60909
00:13:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.021744 30905 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:36.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.022266 30905 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap complete.
00:13:36.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.023229 29127 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:36.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.023907 30905 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent bootstrapping tablet: real 0.044s	user 0.029s	sys 0.004s
00:13:36.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.026907 30905 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:36.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.027508 30905 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: f962932df48b42568f69e73e7e4fd476, State: Initialized, Role: FOLLOWER
00:13:36.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.028057 30905 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.029315 30905 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent starting tablet: real 0.005s	user 0.008s	sys 0.000s
00:13:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.029873 31084 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:36.043 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.2:35557
00:13:36.044 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:36.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:36.044503 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 30295
00:13:36.066 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:36.066 [DEBUG - kudu-nio-0] (Connection.java:239) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] handling channelInactive
00:13:36.067 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state READY due to: connection closed
00:13:36.078 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:36.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.080 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.101 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:36.101 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:36.101 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [0ms] delaying RPC due to: Illegal state: connection is terminated, [11ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [13ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [13ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1941877209(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@96467b8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26b36d3c -> wakeup thread Time-limited test)) after lookup
00:13:36.102 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:36.103 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.103 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.103 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.121 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:36.121 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:36.122 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [0ms] delaying RPC due to: Illegal state: connection is terminated, [11ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [13ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [13ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [35ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [36ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [36ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [54ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1941877209(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@96467b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26b36d3c -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680))) after lookup
00:13:36.122 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:36.124 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.124 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.124 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.181 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:36.182 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:36.182 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=115), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [0ms] delaying RPC due to: Illegal state: connection is terminated, [11ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [13ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [13ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [35ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [36ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [36ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [54ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [55ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [57ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [57ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [114ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1941877209(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@96467b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680) -> (continuation of Deferred@1262411891 after retry RPC@812708056), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26b36d3c -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680) -> (continuation of Deferred@1262411891 after retry RPC@812708056))) after lookup
00:13:36.183 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:36.184 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.184 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.185 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.440 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:36.441 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:36.441 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=374), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [0ms] delaying RPC due to: Illegal state: connection is terminated, [11ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [13ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [13ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [35ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [36ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [36ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [54ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [55ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [57ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [57ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [114ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [115ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [117ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [118ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [370ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [373ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1941877209(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@96467b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680) -> (continuation of Deferred@1262411891 after retry RPC@812708056) -> (continuation of Deferred@1308312315 after retry RPC@2050657708), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26b36d3c -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680) -> (continuation of Deferred@1262411891 after retry RPC@812708056) -> (continuation of Deferred@1308312315 after retry RPC@2050657708))) after lookup
00:13:36.442 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:36.443 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.444 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:36.444 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:37.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.307067 31104 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 5 FOLLOWER]: Starting pre-election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:37.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.307237 31104 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.308462 31104 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.309382 30601 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.310401 30601 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.313083 31015 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476" is_pre_election: true
00:13:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.313431 31015 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 4.
00:13:37.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.314155 30606 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:37.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.314491 31104 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 5 FOLLOWER]: Leader pre-election won for term 6
00:13:37.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.314563 31104 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 5 FOLLOWER]: Starting leader election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:37.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.314594 31104 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 5 FOLLOWER]: Advancing to term 6
00:13:37.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.315605 31104 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:37.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.316042 31104 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 6 election: Requested vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:37.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.316624 31015 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:37.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.316777 31015 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 4 FOLLOWER]: Advancing to term 6
00:13:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.317441 30601 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.318825 31015 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 6.
00:13:37.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.319252 30606 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:37.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.319450 31104 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 6 FOLLOWER]: Leader election won for term 6
00:13:37.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.319769 31104 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 6 LEADER]: Becoming Leader. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Running, Role: LEADER
00:13:37.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.320197 31104 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.322968 29353 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b reported cstate change: term changed from 5 to 6, leader changed from e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2) to 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3). New cstate: current_term: 6 leader_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:37.461 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:37.462 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:37.462 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1395), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [0ms] delaying RPC due to: Illegal state: connection is terminated, [11ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [13ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [13ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [35ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [36ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [36ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [54ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [55ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [57ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [57ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [114ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [115ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [117ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [118ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [370ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [373ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [375ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [377ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [377ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [1391ms] refreshing cache from master, [1391ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [1394ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1941877209(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@96467b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680) -> (continuation of Deferred@1262411891 after retry RPC@812708056) -> (continuation of Deferred@1308312315 after retry RPC@2050657708) -> (continuation of Deferred@675416573 after retry RPC@112965300), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26b36d3c -> wakeup thread Time-limited test -> (continuation of Deferred@388151897 after retry RPC@1322691680) -> (continuation of Deferred@1262411891 after retry RPC@812708056) -> (continuation of Deferred@1308312315 after retry RPC@2050657708) -> (continuation of Deferred@675416573 after retry RPC@112965300))) after lookup
00:13:37.463 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:37.464 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] Successfully connected to peer
00:13:37.496 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0x786851c9, L:/127.0.0.1:46944 - R:/127.28.97.3:44737] using TOKEN/null
00:13:37.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.500373 31015 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 6 FOLLOWER]: Refusing update from remote peer 8f51a58ed1ee47e1ba1b670c49abba2b: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 18. (index mismatch)
00:13:37.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.500682 30601 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b -> Peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Couldn't send request to peer e9fbd638be254004a6d0df9c49d25e3a. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:37.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.501122 31113 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
00:13:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.511180 31148 mvcc.cc:204] Tried to move back new op lower bound from 7317477857274929152 to 7317477856546586624. Current Snapshot: MvccSnapshot[applied={T|T < 7317477857274929152}]
00:13:37.530 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.2:35557
00:13:37.530 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.531445 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:37.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.642975 31157 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:37.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.643236 31157 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:37.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.643326 31157 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.646799 31157 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.646911 31157 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.2
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.650902 31157 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31157
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:37.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:37.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:37.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:37.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.651974 31157 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:37.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.653053 31157 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.660197 31162 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.660192 31163 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.660256 31157 server_base.cc:1061] running on GCE node
00:13:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:37.660192 31168 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:37.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.661244 31157 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:37.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.661854 31157 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:37.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.663067 31157 hybrid_clock.cc:648] HybridClock initialized: now 1786493617663047 us; error 42 us; skew 500 ppm
00:13:37.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.665560 31157 webserver.cc:533] Webserver started at http://127.28.97.2:44183/ using document root <none> and password file <none>
00:13:37.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.666256 31157 fs_manager.cc:362] Metadata directory not provided
00:13:37.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.666353 31157 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:37.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.670457 31157 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
00:13:37.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.673123 31174 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:37.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.674564 31157 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.674721 31157 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.675246 31157 fs_report.cc:389] FS layout report
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/data
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:37.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:37.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:37.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.686545 31157 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:37.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.687390 31157 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:37.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.687631 31157 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:37.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.688354 31157 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.689922 31184 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:37.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.693506 31157 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:37.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.693617 31157 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
00:13:37.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.693707 31157 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.695446 31157 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.695530 31157 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
00:13:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.695950 31184 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap starting.
00:13:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.708683 31157 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.2:35557
00:13:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.708707 31363 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.2:35557 every 8 connection(s)
00:13:37.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.709915 31157 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:37.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.714519 31184 log.cc:826] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Log is configured to *not* fsync() on all Append() calls
00:13:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.718765 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 31157
00:13:37.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.723165 31366 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.723496 31366 heartbeater.cc:461] Registering TS with master...
00:13:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.724378 31365 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.724473 31365 heartbeater.cc:461] Registering TS with master...
00:13:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.724682 31366 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.724689 31365 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.726850 29127 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.728289 29248 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:37.729 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.729017 31364 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.729110 31364 heartbeater.cc:461] Registering TS with master...
00:13:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.729334 31364 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:37.730 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.3:44737
00:13:37.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:13:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.730896 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 30574
00:13:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.739583 31184 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.740079 31184 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap complete.
00:13:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.740398 29353 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.741432 31184 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent bootstrapping tablet: real 0.046s	user 0.026s	sys 0.011s
00:13:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.741575 29353 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.2:54227
00:13:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.745115 31184 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.745659 31184 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Initialized, Role: FOLLOWER
00:13:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.746240 31184 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.747187 31364 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:37.747318 31184 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.001s
00:13:37.751 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] handling channelInactive
00:13:37.751 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state READY due to: connection closed
00:13:37.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:37.752 [DEBUG - Time-limited test] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:37.753 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.753 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.753 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.760 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:37.760 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:37.761 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [1ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@900200459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6ef2e61d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7173cda6 -> wakeup thread Time-limited test)) after lookup
00:13:37.761 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:37.762 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.762 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.762 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.781 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:37.781 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:37.782 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [1ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [9ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [10ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [29ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@900200459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6ef2e61d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7173cda6 -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134))) after lookup
00:13:37.782 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:37.783 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.784 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.784 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.821 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:37.822 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:37.822 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [1ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [9ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [10ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [29ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [30ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [32ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [69ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@900200459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6ef2e61d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7173cda6 -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094))) after lookup
00:13:37.823 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:37.824 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.824 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.824 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.861 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:37.861 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:37.862 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=110), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [1ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [9ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [10ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [29ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [30ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [32ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [69ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [70ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [72ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [109ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@900200459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6ef2e61d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7173cda6 -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219))) after lookup
00:13:37.862 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:37.863 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.864 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:37.864 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:38.321 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:38.322 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:38.322 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=570), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [1ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [9ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [10ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [29ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [30ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [32ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [69ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [70ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [72ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [109ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [110ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [112ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [112ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [566ms] refreshing cache from master, [566ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [569ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@900200459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6ef2e61d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219) -> (continuation of Deferred@464416853 after retry RPC@1593821798), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7173cda6 -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219) -> (continuation of Deferred@464416853 after retry RPC@1593821798))) after lookup
00:13:38.323 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:38.324 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:38.324 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:38.324 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:38.521 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:38.521 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:38.522 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=770), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [1ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [9ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [10ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [29ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [30ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [32ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [69ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [70ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [72ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [109ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [110ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [112ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [112ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [566ms] refreshing cache from master, [566ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [569ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [570ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [572ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [572ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [766ms] refreshing cache from master, [766ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [769ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@900200459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6ef2e61d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219) -> (continuation of Deferred@464416853 after retry RPC@1593821798) -> (continuation of Deferred@800877247 after retry RPC@254814676), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7173cda6 -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219) -> (continuation of Deferred@464416853 after retry RPC@1593821798) -> (continuation of Deferred@800877247 after retry RPC@254814676))) after lookup
00:13:38.523 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:38.524 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:38.524 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:38.524 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.030786 31390 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 8f51a58ed1ee47e1ba1b670c49abba2b)
00:13:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.030959 31390 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:39.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.032114 31390 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:39.033071 30894 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:39.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:39.034205 30894 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:39.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.037695 31289 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "e9fbd638be254004a6d0df9c49d25e3a" is_pre_election: true
00:13:39.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.037992 31289 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f962932df48b42568f69e73e7e4fd476 in term 5.
00:13:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.038800 30893 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b
00:13:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.039045 31390 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 6 FOLLOWER]: Leader pre-election won for term 7
00:13:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.039115 31390 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 8f51a58ed1ee47e1ba1b670c49abba2b)
00:13:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.039157 31390 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 6 FOLLOWER]: Advancing to term 7
00:13:39.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.040119 31390 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:39.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.040529 31390 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 7 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:39.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.041002 31289 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:39.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.041172 31289 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 5 FOLLOWER]: Advancing to term 7
00:13:39.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:39.041757 30894 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:39.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.043260 31289 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f962932df48b42568f69e73e7e4fd476 in term 7.
00:13:39.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.043699 30893 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b
00:13:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.044008 31390 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 7 FOLLOWER]: Leader election won for term 7
00:13:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.044325 31390 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 7 LEADER]: Becoming Leader. State: Replica: f962932df48b42568f69e73e7e4fd476, State: Running, Role: LEADER
00:13:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.044694 31390 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:39.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.047398 29353 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 reported cstate change: term changed from 6 to 7, leader changed from 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3) to f962932df48b42568f69e73e7e4fd476 (127.28.97.1). New cstate: current_term: 7 leader_uuid: "f962932df48b42568f69e73e7e4fd476" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: HEALTHY } } }
00:13:39.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.470005 31289 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 7 FOLLOWER]: Refusing update from remote peer f962932df48b42568f69e73e7e4fd476: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
00:13:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:39.470854 31390 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
00:13:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:39.471737 30894 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 -> Peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Couldn't send request to peer 8f51a58ed1ee47e1ba1b670c49abba2b. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:41.801 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:41.801 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:41.802 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=4050), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [1ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [9ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [10ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [10ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [29ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [30ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [32ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [32ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [69ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [70ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [72ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [72ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [109ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [110ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [112ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [112ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [566ms] refreshing cache from master, [566ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [569ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [570ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [572ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [572ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [766ms] refreshing cache from master, [766ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [769ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [770ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [772ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [772ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [4046ms] refreshing cache from master, [4046ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [4049ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@900200459(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6ef2e61d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219) -> (continuation of Deferred@464416853 after retry RPC@1593821798) -> (continuation of Deferred@800877247 after retry RPC@254814676) -> (continuation of Deferred@786808948 after retry RPC@546644725), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7173cda6 -> wakeup thread Time-limited test -> (continuation of Deferred@528997349 after retry RPC@1895181134) -> (continuation of Deferred@894955753 after retry RPC@1958347094) -> (continuation of Deferred@1754868400 after retry RPC@1044346219) -> (continuation of Deferred@464416853 after retry RPC@1593821798) -> (continuation of Deferred@800877247 after retry RPC@254814676) -> (continuation of Deferred@786808948 after retry RPC@546644725))) after lookup
00:13:41.802 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:41.803 [DEBUG - kudu-nio-6] (Connection.java:757) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] Successfully connected to peer
00:13:41.817 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0xc8577db5, L:/127.0.0.1:50542 - R:/127.28.97.1:41751] using TOKEN/null
00:13:41.838 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.3:44737
00:13:41.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.840360 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:44737
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38861
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:41.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.882148 30894 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 -> Peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Couldn't send request to peer 8f51a58ed1ee47e1ba1b670c49abba2b. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:13:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.956543 31451 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.956781 31451 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:41.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.956827 31451 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.960271 31451 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.960389 31451 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.3
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.964272 31451 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:44737
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38861
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31451
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.965547 31451 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:41.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.966730 31451 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:41.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.973584 31458 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:41.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.973703 31462 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:41.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:41.973582 31459 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:41.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.973831 31451 server_base.cc:1061] running on GCE node
00:13:41.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.974697 31451 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:41.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.975278 31451 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:41.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.976477 31451 hybrid_clock.cc:648] HybridClock initialized: now 1786493621976456 us; error 34 us; skew 500 ppm
00:13:41.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.978452 31451 webserver.cc:533] Webserver started at http://127.28.97.3:38861/ using document root <none> and password file <none>
00:13:41.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.979182 31451 fs_manager.cc:362] Metadata directory not provided
00:13:41.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.979270 31451 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:41.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.982868 31451 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.004s	sys 0.000s
00:13:41.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.985132 31476 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:41.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.986250 31451 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.986403 31451 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:25 on dist-test-slave-3h5h"
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:41.986909 31451 fs_report.cc:389] FS layout report
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/data
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.000762 31451 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.001653 31451 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.001859 31451 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:42.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.002641 31451 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.004271 31485 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.007870 31451 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.008054 31451 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
00:13:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.008129 31451 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:42.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.009842 31451 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:42.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.009922 31451 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
00:13:42.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.010247 31485 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap starting.
00:13:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.022113 31451 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.3:44737
00:13:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.022141 31655 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.3:44737 every 8 connection(s)
00:13:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.023248 31451 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.029511 31485 log.cc:826] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Log is configured to *not* fsync() on all Append() calls
00:13:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.029790 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 31451
00:13:42.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.036968 31657 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.036968 31663 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.037292 31657 heartbeater.cc:461] Registering TS with master...
00:13:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.037940 31657 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.037292 31663 heartbeater.cc:461] Registering TS with master...
00:13:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.039769 31663 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:42.040 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.040748 29127 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:42.041 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.1:41751
00:13:42.042 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:13:42.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.041105 29356 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:42.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.042066 29356 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.3:51765
00:13:42.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.042268 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 30869
00:13:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.048823 31666 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.049237 31666 heartbeater.cc:461] Registering TS with master...
00:13:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.049692 31666 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:42.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.053844 29248 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.057757 31485 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.058252 31485 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap complete.
00:13:42.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.059718 31485 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent bootstrapping tablet: real 0.050s	user 0.038s	sys 0.000s
00:13:42.063 [DEBUG - kudu-nio-6] (Connection.java:239) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] handling channelInactive
00:13:42.063 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state READY due to: connection closed
00:13:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.063087 31485 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.063556 31485 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Initialized, Role: FOLLOWER
00:13:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.064108 31485 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:42.064 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.064898 31657 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:42.065105 31485 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent starting tablet: real 0.005s	user 0.008s	sys 0.000s
00:13:42.066 [DEBUG - Time-limited test] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:42.068 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.068 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.068 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.081 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:42.081 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:42.081 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1253322174(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@433f2c97 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@133b24f -> wakeup thread Time-limited test)) after lookup
00:13:42.082 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:42.084 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.084 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.084 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.101 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:42.101 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:42.101 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [18ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [20ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [20ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1253322174(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@433f2c97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@133b24f -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959))) after lookup
00:13:42.102 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:42.103 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.103 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.103 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.120 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:42.121 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:42.121 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [18ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [20ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [20ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [39ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [39ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [56ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1253322174(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@433f2c97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@133b24f -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419))) after lookup
00:13:42.121 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:42.122 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.123 [DEBUG - kudu-nio-2] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.123 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.181 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:42.181 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:42.181 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=116), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [18ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [20ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [20ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [39ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [39ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [56ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [57ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [59ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [59ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [113ms] refreshing cache from master, [114ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [116ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1253322174(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@433f2c97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419) -> (continuation of Deferred@1542913367 after retry RPC@1810728325), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@133b24f -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419) -> (continuation of Deferred@1542913367 after retry RPC@1810728325))) after lookup
00:13:42.182 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:42.183 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.183 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.183 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.600 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:42.600 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:42.601 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=536), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [18ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [20ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [20ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [39ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [39ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [56ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [57ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [59ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [59ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [113ms] refreshing cache from master, [114ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [116ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [117ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [119ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [119ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [533ms] refreshing cache from master, [533ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [536ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1253322174(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@433f2c97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419) -> (continuation of Deferred@1542913367 after retry RPC@1810728325) -> (continuation of Deferred@1222968012 after retry RPC@1336113144), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@133b24f -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419) -> (continuation of Deferred@1542913367 after retry RPC@1810728325) -> (continuation of Deferred@1222968012 after retry RPC@1336113144))) after lookup
00:13:42.601 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:42.602 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.603 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:42.603 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:43.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.338184 31683 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 7 FOLLOWER]: Starting pre-election (detected failure of leader f962932df48b42568f69e73e7e4fd476)
00:13:43.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.338341 31683 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.339615 31683 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:43.340603 31178 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:43.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:43.341599 31178 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:43.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.345124 31589 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" is_pre_election: true
00:13:43.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.345396 31589 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 6.
00:13:43.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.346118 31176 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a; no voters: f962932df48b42568f69e73e7e4fd476
00:13:43.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.346426 31683 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 7 FOLLOWER]: Leader pre-election won for term 8
00:13:43.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.346501 31683 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 7 FOLLOWER]: Starting leader election (detected failure of leader f962932df48b42568f69e73e7e4fd476)
00:13:43.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.346536 31683 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 7 FOLLOWER]: Advancing to term 8
00:13:43.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.347842 31683 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:43.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.348402 31683 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 8 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:43.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.348959 31589 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:43.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.349100 31589 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 6 FOLLOWER]: Advancing to term 8
00:13:43.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:43.349690 31178 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:43.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.351130 31589 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 8.
00:13:43.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.351497 31176 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a; no voters: f962932df48b42568f69e73e7e4fd476
00:13:43.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.351723 31683 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 8 FOLLOWER]: Leader election won for term 8
00:13:43.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.352030 31683 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 8 LEADER]: Becoming Leader. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Running, Role: LEADER
00:13:43.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.352387 31683 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:43.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.356014 29356 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a reported cstate change: term changed from 7 to 8, leader changed from f962932df48b42568f69e73e7e4fd476 (127.28.97.1) to e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2). New cstate: current_term: 8 leader_uuid: "e9fbd638be254004a6d0df9c49d25e3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:43.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:43.937999 31178 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a -> Peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Couldn't send request to peer f962932df48b42568f69e73e7e4fd476. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:43.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.958024 31589 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 8 FOLLOWER]: Refusing update from remote peer e9fbd638be254004a6d0df9c49d25e3a: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
00:13:43.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:43.958833 31708 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
00:13:43.982 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:43.982 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:43.982 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1917), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [4ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [4ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [18ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [20ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [20ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [39ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [39ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [56ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [57ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [59ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [59ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [113ms] refreshing cache from master, [114ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [116ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [117ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [119ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [119ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [533ms] refreshing cache from master, [533ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [536ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [537ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [539ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [539ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [1914ms] refreshing cache from master, [1914ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [1917ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1253322174(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@433f2c97 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419) -> (continuation of Deferred@1542913367 after retry RPC@1810728325) -> (continuation of Deferred@1222968012 after retry RPC@1336113144) -> (continuation of Deferred@592509951 after retry RPC@1998083290), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@133b24f -> wakeup thread Time-limited test -> (continuation of Deferred@268047277 after retry RPC@331839959) -> (continuation of Deferred@927664772 after retry RPC@535479419) -> (continuation of Deferred@1542913367 after retry RPC@1810728325) -> (continuation of Deferred@1222968012 after retry RPC@1336113144) -> (continuation of Deferred@592509951 after retry RPC@1998083290))) after lookup
00:13:43.983 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:43.984 [DEBUG - kudu-nio-5] (Connection.java:757) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] Successfully connected to peer
00:13:44.010 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0x244354dc, L:/127.0.0.1:57178 - R:/127.28.97.2:35557] using TOKEN/null
00:13:44.040 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.1:41751
00:13:44.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.041448 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:41751
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.156342 31732 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.156584 31732 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.156633 31732 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.160135 31732 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.160260 31732 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.1
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.164163 31732 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:41751
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31732
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.165488 31732 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:44.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.166652 31732 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:44.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.173465 31742 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.173466 31740 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:44.173565 31739 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.173725 31732 server_base.cc:1061] running on GCE node
00:13:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.174155 31732 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.174646 31732 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:44.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.175823 31732 hybrid_clock.cc:648] HybridClock initialized: now 1786493624175799 us; error 48 us; skew 500 ppm
00:13:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.177928 31732 webserver.cc:533] Webserver started at http://127.28.97.1:39491/ using document root <none> and password file <none>
00:13:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.178628 31732 fs_manager.cc:362] Metadata directory not provided
00:13:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.178721 31732 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:44.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.182626 31732 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
00:13:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.185045 31751 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:44.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.186244 31732 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
00:13:44.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.186400 31732 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:44.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.186862 31732 fs_report.cc:389] FS layout report
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/data
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.224727 31732 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.225605 31732 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.225845 31732 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:44.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.226536 31732 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:44.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.227996 31765 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.231767 31732 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.231868 31732 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
00:13:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.232064 31732 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.234025 31732 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.234081 31732 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
00:13:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.234472 31765 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap starting.
00:13:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.247153 31732 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.1:41751
00:13:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.247223 31933 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.1:41751 every 8 connection(s)
00:13:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.248478 31732 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:44.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.249692 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 31732
00:13:44.250 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:44.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.258981 31765 log.cc:826] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Log is configured to *not* fsync() on all Append() calls
00:13:44.260 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:44.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.261142 31938 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:44.261 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.2:35557
00:13:44.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.261682 31938 heartbeater.cc:461] Registering TS with master...
00:13:44.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:44.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.262388 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 31157
00:13:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.268851 31938 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.271493 29248 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:44.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.275249 31934 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.275358 31934 heartbeater.cc:461] Registering TS with master...
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.275532 31937 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.275605 31934 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.275655 31937 heartbeater.cc:461] Registering TS with master...
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.275964 31937 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.277132 29353 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.277395 29127 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.278359 29353 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.1:47119
00:13:44.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:44.285 [DEBUG - kudu-nio-5] (Connection.java:239) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] handling channelInactive
00:13:44.285 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state READY due to: connection closed
00:13:44.285 [DEBUG - Time-limited test] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:44.287 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.287 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.287 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.290515 31765 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:44.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.290984 31765 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap complete.
00:13:44.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.292336 31765 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent bootstrapping tablet: real 0.058s	user 0.039s	sys 0.000s
00:13:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.294659 31765 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.295053 31765 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: f962932df48b42568f69e73e7e4fd476, State: Initialized, Role: FOLLOWER
00:13:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.295538 31765 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.296420 31934 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:44.296547 31765 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent starting tablet: real 0.004s	user 0.007s	sys 0.000s
00:13:44.301 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:44.302 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:44.302 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1461454818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6217375f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@262aca99 -> wakeup thread Time-limited test)) after lookup
00:13:44.303 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:44.304 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.304 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.304 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.321 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:44.321 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:44.321 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [17ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [19ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [19ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1461454818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6217375f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@262aca99 -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350))) after lookup
00:13:44.322 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:44.323 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.323 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.323 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.361 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:44.362 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:44.362 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [17ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [19ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [19ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [38ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [38ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [76ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1461454818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6217375f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@262aca99 -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346))) after lookup
00:13:44.362 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:44.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.364 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.364 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.501 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:44.501 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:44.502 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=216), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [17ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [19ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [19ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [38ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [38ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [76ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [77ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [79ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [79ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [216ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1461454818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6217375f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346) -> (continuation of Deferred@107289665 after retry RPC@1287588003), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@262aca99 -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346) -> (continuation of Deferred@107289665 after retry RPC@1287588003))) after lookup
00:13:44.503 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:44.504 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.504 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:44.504 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:45.001 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:45.001 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:45.002 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=717), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [17ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [19ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [19ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [38ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [38ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [76ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [77ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [79ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [79ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [216ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [217ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [219ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [219ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [713ms] refreshing cache from master, [713ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [716ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1461454818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6217375f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346) -> (continuation of Deferred@107289665 after retry RPC@1287588003) -> (continuation of Deferred@1631776818 after retry RPC@1730681347), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@262aca99 -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346) -> (continuation of Deferred@107289665 after retry RPC@1287588003) -> (continuation of Deferred@1631776818 after retry RPC@1730681347))) after lookup
00:13:45.002 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:45.003 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:45.003 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:45.004 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.540601 31955 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 8 FOLLOWER]: Starting pre-election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:45.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.540818 31955 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.542057 31955 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:45.543020 31477 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:45.544204 31477 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.547786 31859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476" is_pre_election: true
00:13:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.548182 31859 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 7.
00:13:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.549108 31481 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.549530 31955 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 8 FOLLOWER]: Leader pre-election won for term 9
00:13:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.549599 31955 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 8 FOLLOWER]: Starting leader election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.549630 31955 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 8 FOLLOWER]: Advancing to term 9
00:13:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.550530 31955 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.550896 31955 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 9 election: Requested vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.551429 31859 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.551580 31859 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 7 FOLLOWER]: Advancing to term 9
00:13:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:45.552309 31477 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.554005 31859 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 9.
00:13:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.554368 31481 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.554592 31955 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 9 FOLLOWER]: Leader election won for term 9
00:13:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.554872 31955 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 9 LEADER]: Becoming Leader. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Running, Role: LEADER
00:13:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.555214 31955 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:45.557920 29353 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b reported cstate change: term changed from 8 to 9, leader changed from e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2) to 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3). New cstate: current_term: 9 leader_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:46.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:46.146340 31859 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 9 FOLLOWER]: Refusing update from remote peer 8f51a58ed1ee47e1ba1b670c49abba2b: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
00:13:46.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:46.147310 31984 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
00:13:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:46.150024 31477 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b -> Peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Couldn't send request to peer e9fbd638be254004a6d0df9c49d25e3a. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:46.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:46.152686 31992 mvcc.cc:204] Tried to move back new op lower bound from 7317477884050755584 to 7317477881255534592. Current Snapshot: MvccSnapshot[applied={T|T < 7317477883957297152}]
00:13:46.942 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:46.942 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:46.942 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2657), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [12ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [16ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [17ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [19ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [19ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [38ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [38ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [76ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [77ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [79ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [79ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [213ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [216ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [217ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [219ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [219ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [713ms] refreshing cache from master, [713ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [716ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [717ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [719ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [719ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2653ms] refreshing cache from master, [2653ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [2656ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1461454818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@6217375f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346) -> (continuation of Deferred@107289665 after retry RPC@1287588003) -> (continuation of Deferred@1631776818 after retry RPC@1730681347) -> (continuation of Deferred@1384762719 after retry RPC@1976970511), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@262aca99 -> wakeup thread Time-limited test -> (continuation of Deferred@94062566 after retry RPC@1972710350) -> (continuation of Deferred@1008911275 after retry RPC@1622492346) -> (continuation of Deferred@107289665 after retry RPC@1287588003) -> (continuation of Deferred@1631776818 after retry RPC@1730681347) -> (continuation of Deferred@1384762719 after retry RPC@1976970511))) after lookup
00:13:46.943 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:46.944 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] Successfully connected to peer
00:13:46.974 [DEBUG - kudu-nio-4] (Negotiator.java:918) Authenticated connection [id: 0xbc2f41ae, L:/127.0.0.1:39564 - R:/127.28.97.3:44737] using TOKEN/null
00:13:47.000 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.2:35557
00:13:47.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.001223 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:47.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.119449 32014 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:47.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.119699 32014 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:47.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.119763 32014 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:47.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.123271 32014 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:47.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.123438 32014 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.2
00:13:47.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.127586 32014 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:47.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:47.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:47.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:47.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.32014
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:47.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.129145 32014 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:47.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.130497 32014 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.137799 32029 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.137925 32014 server_base.cc:1061] running on GCE node
00:13:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.137794 32024 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:47.137792 32025 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.138832 32014 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:47.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.139511 32014 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:47.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.140702 32014 hybrid_clock.cc:648] HybridClock initialized: now 1786493627140668 us; error 50 us; skew 500 ppm
00:13:47.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.142807 32014 webserver.cc:533] Webserver started at http://127.28.97.2:44183/ using document root <none> and password file <none>
00:13:47.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.143438 32014 fs_manager.cc:362] Metadata directory not provided
00:13:47.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.143530 32014 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:47.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.147024 32014 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
00:13:47.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.149291 32041 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.150384 32014 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.150521 32014 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.150965 32014 fs_report.cc:389] FS layout report
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/data
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.172691 32014 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.173522 32014 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.173774 32014 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:47.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.174474 32014 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.175932 32057 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:47.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.179824 32014 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:47.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.179915 32014 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
00:13:47.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.179986 32014 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:47.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.181553 32014 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:47.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.181625 32014 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
00:13:47.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.182005 32057 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap starting.
00:13:47.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.193832 32014 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.2:35557
00:13:47.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.193909 32224 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.2:35557 every 8 connection(s)
00:13:47.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.194936 32014 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.199169 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 32014
00:13:47.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.203037 32057 log.cc:826] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Log is configured to *not* fsync() on all Append() calls
00:13:47.208 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:47.209 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.3:44737
00:13:47.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:13:47.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.213155 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 31451
00:13:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.235603 32233 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.235934 32233 heartbeater.cc:461] Registering TS with master...
00:13:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.236650 32233 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.239073 29127 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.240548 32231 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:47.244 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] handling channelInactive
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.240646 32231 heartbeater.cc:461] Registering TS with master...
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.240736 32234 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:47.244 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state READY due to: connection closed
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.240797 32234 heartbeater.cc:461] Registering TS with master...
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.240916 32231 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.240975 32234 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.241421 32057 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.241844 32057 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap complete.
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.241938 29247 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.241990 29353 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.243078 29353 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.2:47145
00:13:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.243269 32057 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent bootstrapping tablet: real 0.062s	user 0.037s	sys 0.008s
00:13:47.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:47.246 [DEBUG - Time-limited test] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.245796 32057 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.246109 32057 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Initialized, Role: FOLLOWER
00:13:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.246580 32057 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:47.247 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.247 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.247 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.247933 32231 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:47.249529 32057 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
00:13:47.260 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:47.260 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:47.260 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@646616334(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@62f06a85 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b4a74eb -> wakeup thread Time-limited test)) after lookup
00:13:47.261 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:47.262 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.262 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.262 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.281 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:47.281 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:47.281 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [16ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [17ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@646616334(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@62f06a85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b4a74eb -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798))) after lookup
00:13:47.282 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:47.283 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.283 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.283 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.320 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:47.321 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:47.321 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [16ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [17ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [38ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [38ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [75ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@646616334(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@62f06a85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b4a74eb -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702))) after lookup
00:13:47.321 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:47.322 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.322 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.323 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.461 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:47.461 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:47.461 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=216), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [16ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [17ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [38ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [38ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [75ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [76ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [77ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [78ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [212ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [215ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@646616334(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@62f06a85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702) -> (continuation of Deferred@688221937 after retry RPC@1712520673), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b4a74eb -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702) -> (continuation of Deferred@688221937 after retry RPC@1712520673))) after lookup
00:13:47.462 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:47.463 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.463 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.464 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.581 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:47.581 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:47.582 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=336), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [16ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [17ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [38ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [38ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [75ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [76ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [77ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [78ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [212ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [215ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [217ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [219ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [219ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [333ms] refreshing cache from master, [333ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [336ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@646616334(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@62f06a85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702) -> (continuation of Deferred@688221937 after retry RPC@1712520673) -> (continuation of Deferred@1120567486 after retry RPC@1350682814), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b4a74eb -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702) -> (continuation of Deferred@688221937 after retry RPC@1712520673) -> (continuation of Deferred@1120567486 after retry RPC@1350682814))) after lookup
00:13:47.582 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:47.583 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.583 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state CONNECTING due to: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:47.583 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737
00:13:48.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.500437 32256 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 8f51a58ed1ee47e1ba1b670c49abba2b)
00:13:48.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.500608 32256 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.501864 32256 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:48.502718 31756 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:48.503638 31756 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.507282 32153 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "e9fbd638be254004a6d0df9c49d25e3a" is_pre_election: true
00:13:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.507543 32153 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f962932df48b42568f69e73e7e4fd476 in term 8.
00:13:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.508262 31754 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b
00:13:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.508519 32256 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 9 FOLLOWER]: Leader pre-election won for term 10
00:13:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.508622 32256 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 8f51a58ed1ee47e1ba1b670c49abba2b)
00:13:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.508671 32256 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 9 FOLLOWER]: Advancing to term 10
00:13:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.509620 32256 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.510072 32256 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 10 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.510495 32153 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.510628 32153 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 8 FOLLOWER]: Advancing to term 10
00:13:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:48.511265 31756 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111)
00:13:48.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.512620 32153 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f962932df48b42568f69e73e7e4fd476 in term 10.
00:13:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.513051 31754 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e9fbd638be254004a6d0df9c49d25e3a, f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b
00:13:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.513289 32256 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Leader election won for term 10
00:13:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.513550 32256 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 LEADER]: Becoming Leader. State: Replica: f962932df48b42568f69e73e7e4fd476, State: Running, Role: LEADER
00:13:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.513906 32256 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.516914 29353 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 reported cstate change: term changed from 9 to 10, leader changed from 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3) to f962932df48b42568f69e73e7e4fd476 (127.28.97.1). New cstate: current_term: 10 leader_uuid: "f962932df48b42568f69e73e7e4fd476" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: HEALTHY } } }
00:13:48.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.966502 32153 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 10 FOLLOWER]: Refusing update from remote peer f962932df48b42568f69e73e7e4fd476: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 33. (index mismatch)
00:13:48.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:48.967301 32256 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
00:13:48.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:48.976537 31756 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 -> Peer 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737): Couldn't send request to peer 8f51a58ed1ee47e1ba1b670c49abba2b. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.3:44737: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:49.241 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:49.241 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:49.242 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1996), Traces: [0ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [2ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [2ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [16ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [17ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [17ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [35ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [37ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [38ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [38ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [75ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [76ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [77ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [78ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [212ms] refreshing cache from master, [213ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [215ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [217ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [219ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [219ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [333ms] refreshing cache from master, [333ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [336ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [337ms] sending RPC to server 8f51a58ed1ee47e1ba1b670c49abba2b, [338ms] received response from server 8f51a58ed1ee47e1ba1b670c49abba2b: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [338ms] delaying RPC due to: Network error: Failed to connect to peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737): Connection refused: /127.28.97.3:44737, [1992ms] refreshing cache from master, [1993ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [1995ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@646616334(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@62f06a85 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702) -> (continuation of Deferred@688221937 after retry RPC@1712520673) -> (continuation of Deferred@1120567486 after retry RPC@1350682814) -> (continuation of Deferred@2071696458 after retry RPC@435584979), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4b4a74eb -> wakeup thread Time-limited test -> (continuation of Deferred@1218122330 after retry RPC@1357830798) -> (continuation of Deferred@884418764 after retry RPC@304856702) -> (continuation of Deferred@688221937 after retry RPC@1712520673) -> (continuation of Deferred@1120567486 after retry RPC@1350682814) -> (continuation of Deferred@2071696458 after retry RPC@435584979))) after lookup
00:13:49.242 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:49.243 [DEBUG - kudu-nio-3] (Connection.java:757) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] Successfully connected to peer
00:13:49.260 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x818249e4, L:/127.0.0.1:57870 - R:/127.28.97.1:41751] using TOKEN/null
00:13:49.280 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.3:44737
00:13:49.280 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.281028 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:44737
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38861
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.404932 32297 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.405200 32297 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.405262 32297 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:49.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.408670 32297 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.408831 32297 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.3
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.412952 32297 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.3:44737
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.3
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38861
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.32297
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.3
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/logs
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.414508 32297 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.415768 32297 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.422713 32307 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.422789 32311 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.422945 32297 server_base.cc:1061] running on GCE node
00:13:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:49.422716 32304 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.423508 32297 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:49.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.424094 32297 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.425258 32297 hybrid_clock.cc:648] HybridClock initialized: now 1786493629425243 us; error 34 us; skew 500 ppm
00:13:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.427119 32297 webserver.cc:533] Webserver started at http://127.28.97.3:38861/ using document root <none> and password file <none>
00:13:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.427657 32297 fs_manager.cc:362] Metadata directory not provided
00:13:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.427709 32297 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.431046 32297 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.001s
00:13:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.433037 32322 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.434100 32297 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.434201 32297 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:25 on dist-test-slave-3h5h"
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.434553 32297 fs_report.cc:389] FS layout report
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/wal
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/data
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.455312 32297 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.456173 32297 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.456332 32297 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:49.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.456969 32297 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:49.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.458328 32334 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.461875 32297 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.461937 32297 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
00:13:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.461989 32297 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:49.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.463728 32297 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:49.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.463801 32297 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
00:13:49.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.464197 32334 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap starting.
00:13:49.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.476130 32297 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.3:44737
00:13:49.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.476161 32504 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.3:44737 every 8 connection(s)
00:13:49.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.477238 32297 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-2/data/info.pb
00:13:49.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.484012 32334 log.cc:826] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Log is configured to *not* fsync() on all Append() calls
00:13:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.485894 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 32297
00:13:49.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.488525 32505 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.488808 32505 heartbeater.cc:461] Registering TS with master...
00:13:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.489557 32505 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.489902 32508 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.489980 32508 heartbeater.cc:461] Registering TS with master...
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.490156 32508 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.491750 29248 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.492086 29353 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.493304 29353 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.3:36617
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.494479 32507 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.494566 32507 heartbeater.cc:461] Registering TS with master...
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.494699 32507 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:49.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.495635 29127 ts_manager.cc:194] Re-registered known tserver with Master: 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737)
00:13:49.500 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:49.501 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.1:41751
00:13:49.501 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:13:49.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.501722 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 31732
00:13:49.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.518774 32334 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:49.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.519170 32334 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Bootstrap complete.
00:13:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.520289 32334 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent bootstrapping tablet: real 0.056s	user 0.030s	sys 0.016s
00:13:49.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:49.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.522835 32334 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:49.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.523211 32334 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Initialized, Role: FOLLOWER
00:13:49.523 [DEBUG - kudu-nio-3] (Connection.java:239) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] handling channelInactive
00:13:49.523 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state READY due to: connection closed
00:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.523984 32334 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.524821 32505 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:49.524976 32334 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b: Time spent starting tablet: real 0.005s	user 0.008s	sys 0.000s
00:13:49.539 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:49.540 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.540 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.540 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.560 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:49.561 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:49.561 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=3, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [17ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [17ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@933765372(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@1d8ad93d -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b907f79 -> wakeup thread Time-limited test)) after lookup
00:13:49.561 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:49.562 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.563 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.563 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.581 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:49.581 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:49.582 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=5, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [17ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [17ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [40ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [40ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [58ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@933765372(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@1d8ad93d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b907f79 -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768))) after lookup
00:13:49.582 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:49.583 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.583 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.584 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.601 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:49.601 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:49.601 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=7, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [17ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [17ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [40ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [40ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [58ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [59ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [61ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [61ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [77ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@933765372(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@1d8ad93d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b907f79 -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651))) after lookup
00:13:49.602 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:49.603 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.603 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.603 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.800 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:49.801 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:49.801 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=9, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=278), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [17ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [17ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [40ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [40ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [58ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [59ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [61ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [61ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [77ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [78ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [80ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [80ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [274ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [277ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@933765372(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@1d8ad93d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651) -> (continuation of Deferred@827821179 after retry RPC@1766640145), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b907f79 -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651) -> (continuation of Deferred@827821179 after retry RPC@1766640145))) after lookup
00:13:49.802 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:49.802 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.803 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:49.803 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:50.181 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:50.182 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)[L]]}]
00:13:50.182 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=11, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=659), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [17ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [17ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [40ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [40ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [58ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [59ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [61ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [61ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [77ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [78ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [80ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [80ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [274ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [277ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [278ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [280ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [280ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [654ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [658ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@933765372(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@1d8ad93d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651) -> (continuation of Deferred@827821179 after retry RPC@1766640145) -> (continuation of Deferred@483279280 after retry RPC@324869242), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b907f79 -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651) -> (continuation of Deferred@827821179 after retry RPC@1766640145) -> (continuation of Deferred@483279280 after retry RPC@324869242))) after lookup
00:13:50.182 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] connecting to peer
00:13:50.183 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:50.183 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] cleaning up while in state CONNECTING due to: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:50.184 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751
00:13:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.779611 32524 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 10 FOLLOWER]: Starting pre-election (detected failure of leader f962932df48b42568f69e73e7e4fd476)
00:13:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.779811 32524 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.781068 32524 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:50.782115 32050 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:50.783070 32050 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:50.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.786619 32432 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" is_pre_election: true
00:13:50.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.786965 32432 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 9.
00:13:50.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.787796 32044 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a; no voters: f962932df48b42568f69e73e7e4fd476
00:13:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.788062 32524 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 10 FOLLOWER]: Leader pre-election won for term 11
00:13:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.788157 32524 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 10 FOLLOWER]: Starting leader election (detected failure of leader f962932df48b42568f69e73e7e4fd476)
00:13:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.788203 32524 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 10 FOLLOWER]: Advancing to term 11
00:13:50.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.789182 32524 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:50.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.789641 32524 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 11 election: Requested vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.790104 32432 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "e9fbd638be254004a6d0df9c49d25e3a" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b"
00:13:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.790274 32432 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 9 FOLLOWER]: Advancing to term 11
00:13:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:50.790776 32050 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111)
00:13:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.792169 32432 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9fbd638be254004a6d0df9c49d25e3a in term 11.
00:13:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.792544 32044 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a; no voters: f962932df48b42568f69e73e7e4fd476
00:13:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.792747 32524 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 11 FOLLOWER]: Leader election won for term 11
00:13:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.793031 32524 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 11 LEADER]: Becoming Leader. State: Replica: e9fbd638be254004a6d0df9c49d25e3a, State: Running, Role: LEADER
00:13:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.793396 32524 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:50.795884 29353 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a reported cstate change: term changed from 10 to 11, leader changed from f962932df48b42568f69e73e7e4fd476 (127.28.97.1) to e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2). New cstate: current_term: 11 leader_uuid: "e9fbd638be254004a6d0df9c49d25e3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.178542 32432 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Refusing update from remote peer e9fbd638be254004a6d0df9c49d25e3a: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 37. (index mismatch)
00:13:51.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.179352 32524 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
00:13:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.189445 32050 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a -> Peer f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751): Couldn't send request to peer f962932df48b42568f69e73e7e4fd476. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.1:41751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:51.720 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:51.721 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:51.721 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=13, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2198), Traces: [0ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [0ms] delaying RPC due to: Illegal state: connection is terminated, [15ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [17ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [17ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [37ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [38ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [40ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [40ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [55ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [58ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [59ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [61ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [61ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [77ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [78ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [80ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [80ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [274ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [277ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [278ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [280ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [280ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [654ms] refreshing cache from master, [655ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [658ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [659ms] sending RPC to server f962932df48b42568f69e73e7e4fd476, [661ms] received response from server f962932df48b42568f69e73e7e4fd476: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [661ms] delaying RPC due to: Network error: Failed to connect to peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751): Connection refused: /127.28.97.1:41751, [2194ms] refreshing cache from master, [2194ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [2197ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@933765372(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@1d8ad93d -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651) -> (continuation of Deferred@827821179 after retry RPC@1766640145) -> (continuation of Deferred@483279280 after retry RPC@324869242) -> (continuation of Deferred@514854359 after retry RPC@1825902387), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@b907f79 -> wakeup thread Time-limited test -> (continuation of Deferred@399327207 after retry RPC@531554768) -> (continuation of Deferred@1557484008 after retry RPC@874355651) -> (continuation of Deferred@827821179 after retry RPC@1766640145) -> (continuation of Deferred@483279280 after retry RPC@324869242) -> (continuation of Deferred@514854359 after retry RPC@1825902387))) after lookup
00:13:51.722 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:51.723 [DEBUG - kudu-nio-2] (Connection.java:757) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] Successfully connected to peer
00:13:51.738 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x17f8b80d, L:/127.0.0.1:40648 - R:/127.28.97.2:35557] using TOKEN/null
00:13:51.758 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.1:41751
00:13:51.758 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.759301 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:41751
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
00:13:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:51.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.873601 32589 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:51.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.873870 32589 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:51.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.873919 32589 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.877648 32589 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.877815 32589 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.1
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.881983 32589 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.1:41751
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.1
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.32589
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.1
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/logs
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.883545 32589 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.884903 32589 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.892143 32601 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.892130 32595 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:51.892145 32597 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.892566 32589 server_base.cc:1061] running on GCE node
00:13:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.893064 32589 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.893749 32589 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.894985 32589 hybrid_clock.cc:648] HybridClock initialized: now 1786493631894962 us; error 38 us; skew 500 ppm
00:13:51.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.897003 32589 webserver.cc:533] Webserver started at http://127.28.97.1:39491/ using document root <none> and password file <none>
00:13:51.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.897645 32589 fs_manager.cc:362] Metadata directory not provided
00:13:51.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.897753 32589 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:51.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.901379 32589 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.001s	sys 0.003s
00:13:51.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.903555 32614 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:51.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.904707 32589 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.904877 32589 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.905335 32589 fs_report.cc:389] FS layout report
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/wal
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/data
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:51.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:51.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.916443 32589 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:51.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.917234 32589 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:51.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.917459 32589 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:51.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.918111 32589 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.919467 32623 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.923123 32589 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.923206 32589 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
00:13:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.923269 32589 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.925235 32589 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.925292 32589 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.002s	sys 0.000s
00:13:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.925803 32623 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap starting.
00:13:51.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.938014 32589 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.1:41751
00:13:51.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.938032   313 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.1:41751 every 8 connection(s)
00:13:51.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.939280 32589 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-0/data/info.pb
00:13:51.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.944404 32623 log.cc:826] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Log is configured to *not* fsync() on all Append() calls
00:13:51.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:51.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.946286 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 32589
00:13:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.951704   314 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.951987   314 heartbeater.cc:461] Registering TS with master...
00:13:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.951704   322 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.952495   322 heartbeater.cc:461] Registering TS with master...
00:13:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.952637   314 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.952714   322 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.954595   316 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.954818   316 heartbeater.cc:461] Registering TS with master...
00:13:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.954799 29356 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:51.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.954847 29248 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:51.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.955436   316 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:51.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.955837 29356 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.1:56389
00:13:51.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.956629 29126 ts_manager.cc:194] Re-registered known tserver with Master: f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:51.957 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:51.957 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.97.2:35557
00:13:51.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.958465 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 32014
00:13:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.982407 32623 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.983248 32623 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Bootstrap complete.
00:13:51.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.985303 32623 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent bootstrapping tablet: real 0.060s	user 0.039s	sys 0.008s
00:13:51.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.988970 32623 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:51.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.989547 32623 raft_consensus.cc:740] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: f962932df48b42568f69e73e7e4fd476, State: Initialized, Role: FOLLOWER
00:13:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.990193 32623 consensus_queue.cc:260] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:51.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.991093   314 heartbeater.cc:499] Master 127.28.97.60:44689 was elected leader, sending a full tablet report...
00:13:51.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:51.991410 32623 ts_tablet_manager.cc:1434] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
00:13:51.993 [DEBUG - kudu-nio-2] (Connection.java:239) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] handling channelInactive
00:13:51.993 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state READY due to: connection closed
00:13:51.994 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:51.995 [DEBUG - Time-limited test] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:51.996 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:51.996 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:51.996 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.021 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:52.021 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:52.021 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [26ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1546704608(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@3f9a9a44 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@752384ed -> wakeup thread Time-limited test)) after lookup
00:13:52.022 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:52.023 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.023 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.023 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.040 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:52.041 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:52.041 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [26ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [28ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [29ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [29ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [46ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1546704608(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@3f9a9a44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@752384ed -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659))) after lookup
00:13:52.041 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:52.042 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.042 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.043 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.081 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:52.081 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:52.081 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [26ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [28ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [29ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [29ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [46ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [48ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [49ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1546704608(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@3f9a9a44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@752384ed -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059))) after lookup
00:13:52.082 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:52.083 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.083 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.083 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.102 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:52.103 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:52.103 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=109), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [26ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [28ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [29ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [29ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [46ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [48ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [49ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [88ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [89ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [89ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [108ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1546704608(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@3f9a9a44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059) -> (continuation of Deferred@1403464668 after retry RPC@1134119968), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@752384ed -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059) -> (continuation of Deferred@1403464668 after retry RPC@1134119968))) after lookup
00:13:52.104 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:52.105 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.105 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.105 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.601 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:52.602 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737),e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)[L],f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:52.602 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=607), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [26ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [28ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [29ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [29ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [46ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [48ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [49ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [88ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [89ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [89ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [108ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [109ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [111ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [112ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [604ms] refreshing cache from master, [604ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [607ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1546704608(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@3f9a9a44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059) -> (continuation of Deferred@1403464668 after retry RPC@1134119968) -> (continuation of Deferred@519875137 after retry RPC@395120114), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@752384ed -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059) -> (continuation of Deferred@1403464668 after retry RPC@1134119968) -> (continuation of Deferred@519875137 after retry RPC@395120114))) after lookup
00:13:52.602 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] connecting to peer
00:13:52.603 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.603 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] cleaning up while in state CONNECTING due to: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:52.604 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557) for tablet 577804064edd41e4802dc27206597d97: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557
00:13:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.258725   344 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Starting pre-election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.258878   344 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.260105   344 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:53.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:53.260812 32326 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:53.261866 32326 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.262032   353 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:13:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.262182   353 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.263145   353 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3:44737), e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:53.263962 32615 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:53.265448 32615 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.266420 32709 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476" is_pre_election: true
00:13:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.266662 32709 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 10.
00:13:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.267395 32432 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "f962932df48b42568f69e73e7e4fd476" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" is_pre_election: true
00:13:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.267423 32329 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.267602 32432 raft_consensus.cc:2393] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate f962932df48b42568f69e73e7e4fd476 in current term 11: Already voted for candidate e9fbd638be254004a6d0df9c49d25e3a in this term.
00:13:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.267699   344 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Leader pre-election won for term 12
00:13:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.267750   344 raft_consensus.cc:493] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Starting leader election (detected failure of leader e9fbd638be254004a6d0df9c49d25e3a)
00:13:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.267789   344 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 11 FOLLOWER]: Advancing to term 12
00:13:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.268067 32618 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f962932df48b42568f69e73e7e4fd476; no voters: 8f51a58ed1ee47e1ba1b670c49abba2b, e9fbd638be254004a6d0df9c49d25e3a
00:13:53.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.268339   353 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 10 FOLLOWER]: Advancing to term 11
00:13:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.268826   344 raft_consensus.cc:515] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.269274   344 leader_election.cc:290] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 12 election: Requested vote from peers e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557), f962932df48b42568f69e73e7e4fd476 (127.28.97.1:41751)
00:13:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.269891 32709 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "577804064edd41e4802dc27206597d97" candidate_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "f962932df48b42568f69e73e7e4fd476"
00:13:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:53.270284 32326 leader_election.cc:336] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111)
00:13:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.270569   353 raft_consensus.cc:2749] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 11 FOLLOWER]: Leader pre-election lost for term 11. Reason: could not achieve majority
00:13:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.270774 32709 raft_consensus.cc:3060] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 11 FOLLOWER]: Advancing to term 12
00:13:53.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.271598 32709 raft_consensus.cc:2468] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f51a58ed1ee47e1ba1b670c49abba2b in term 12.
00:13:53.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.271924 32329 leader_election.cc:304] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8f51a58ed1ee47e1ba1b670c49abba2b, f962932df48b42568f69e73e7e4fd476; no voters: e9fbd638be254004a6d0df9c49d25e3a
00:13:53.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.272150   344 raft_consensus.cc:2804] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 12 FOLLOWER]: Leader election won for term 12
00:13:53.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.272435   344 raft_consensus.cc:697] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [term 12 LEADER]: Becoming Leader. State: Replica: 8f51a58ed1ee47e1ba1b670c49abba2b, State: Running, Role: LEADER
00:13:53.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.272794   344 consensus_queue.cc:237] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.275437 29356 catalog_manager.cc:5719] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b reported cstate change: term changed from 11 to 12, leader changed from e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2) to 8f51a58ed1ee47e1ba1b670c49abba2b (127.28.97.3). New cstate: current_term: 12 leader_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } health_report { overall_health: UNKNOWN } } }
00:13:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:53.658481 32326 consensus_peers.cc:597] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b -> Peer e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557): Couldn't send request to peer e9fbd638be254004a6d0df9c49d25e3a. Status: Network error: Client connection negotiation failed: client connection to 127.28.97.2:35557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:13:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.876276 32709 raft_consensus.cc:1275] T 577804064edd41e4802dc27206597d97 P f962932df48b42568f69e73e7e4fd476 [term 12 FOLLOWER]: Refusing update from remote peer 8f51a58ed1ee47e1ba1b670c49abba2b: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 41. (index mismatch)
00:13:53.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:53.877134   344 consensus_queue.cc:1048] T 577804064edd41e4802dc27206597d97 P 8f51a58ed1ee47e1ba1b670c49abba2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
00:13:54.142 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 577804064edd41e4802dc27206597d97 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642' with partition [<start>, <end>)
00:13:54.142 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=577804064edd41e4802dc27206597d97@[8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)[L],e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557),f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)]}]
00:13:54.142 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=577804064edd41e4802dc27206597d97, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2148), Traces: [0ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [2ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [26ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [28ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [29ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [29ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [46ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [47ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [48ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [49ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [84ms] refreshing cache from master, [84ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [87ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [88ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [89ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [89ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [108ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [109ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [111ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [112ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [604ms] refreshing cache from master, [604ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [607ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, [608ms] sending RPC to server e9fbd638be254004a6d0df9c49d25e3a, [610ms] received response from server e9fbd638be254004a6d0df9c49d25e3a: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [610ms] delaying RPC due to: Network error: Failed to connect to peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557): Connection refused: /127.28.97.2:35557, [2144ms] refreshing cache from master, [2144ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.97.60:44689, [2147ms] Sub RPC GetTableLocations: received response from server master-127.28.97.60:44689: OK, deferred=Deferred@1546704608(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f102436b560@3f9a9a44 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059) -> (continuation of Deferred@1403464668 after retry RPC@1134119968) -> (continuation of Deferred@519875137 after retry RPC@395120114) -> (continuation of Deferred@1146177915 after retry RPC@1003241118), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@752384ed -> wakeup thread Time-limited test -> (continuation of Deferred@1877459278 after retry RPC@109716659) -> (continuation of Deferred@1728844981 after retry RPC@821300059) -> (continuation of Deferred@1403464668 after retry RPC@1134119968) -> (continuation of Deferred@519875137 after retry RPC@395120114) -> (continuation of Deferred@1146177915 after retry RPC@1003241118))) after lookup
00:13:54.143 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] connecting to peer
00:13:54.144 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] Successfully connected to peer
00:13:54.161 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xec542e74, L:/127.0.0.1:48188 - R:/127.28.97.3:44737] using TOKEN/null
00:13:54.184 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.97.2:35557
00:13:54.184 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.185232 29060 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:13:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:13:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.299304   391 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:13:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.299523   391 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:13:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.299568   391 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:13:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.303200   391 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:13:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.303308   391 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.97.2
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.307260   391 tablet_server_runner.cc:78] Tablet server non-default flags:
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.97.20:41515
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:13:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.97.2:35557
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.97.2
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44183
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.97.62:40027,127.28.97.61:39769,127.28.97.60:44689
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.391
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.97.2
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/logs
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7ed5b56c096f54e2f926bf94ceb7fc44c0c1329
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 23:51:01 UTC on bdcb31816ec0
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17636
00:13:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.308398   391 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:13:54.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.309651   391 file_cache.cc:504] Constructed file cache file cache with capacity 419430
00:13:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.316396   403 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.316396   400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 00:13:54.316426   405 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:13:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.316722   391 server_base.cc:1061] running on GCE node
00:13:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.317507   391 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:13:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.318033   391 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:13:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.319233   391 hybrid_clock.cc:648] HybridClock initialized: now 1786493634319211 us; error 51 us; skew 500 ppm
00:13:54.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.321554   391 webserver.cc:533] Webserver started at http://127.28.97.2:44183/ using document root <none> and password file <none>
00:13:54.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.322391   391 fs_manager.cc:362] Metadata directory not provided
00:13:54.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.322471   391 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:13:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.327396   391 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.004s	sys 0.000s
00:13:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.330061   411 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:13:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.332413   391 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
00:13:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.332536   391 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data,/tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9fbd638be254004a6d0df9c49d25e3a"
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 00:13:24 on dist-test-slave-3h5h"
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.333115   391 fs_report.cc:389] FS layout report
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/wal
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/data
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:13:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:13:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.356315   391 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:13:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.357322   391 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:13:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.357545   391 kserver.cc:163] Server-wide thread pool size limit: 3276
00:13:54.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.358683   391 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:13:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.363368   422 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
00:13:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.369448   391 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:13:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.369575   391 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
00:13:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.369663   391 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:13:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.371415   391 ts_tablet_manager.cc:616] Registered 1 tablets
00:13:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.371490   391 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
00:13:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.371826   422 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap starting.
00:13:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.385735   600 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.97.2:35557 every 8 connection(s)
00:13:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.385732   391 rpc_server.cc:307] RPC server started. Bound to: 127.28.97.2:35557
00:13:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.387446   391 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskC8xxxa/test-tmp/mini-kudu-cluster7202702295587835858/ts-1/data/info.pb
00:13:54.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.393085 29060 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu as pid 391
00:13:54.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:13:54.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.399096   422 log.cc:826] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Log is configured to *not* fsync() on all Append() calls
00:13:54.403 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.406574   606 heartbeater.cc:344] Connected to a master server at 127.28.97.62:40027
00:13:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.406566   608 heartbeater.cc:344] Connected to a master server at 127.28.97.61:39769
00:13:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.406865   606 heartbeater.cc:461] Registering TS with master...
00:13:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.406865   608 heartbeater.cc:461] Registering TS with master...
00:13:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.407557   608 heartbeater.cc:507] Master 127.28.97.61:39769 requested a full tablet report, sending...
00:13:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.407557   606 heartbeater.cc:507] Master 127.28.97.62:40027 requested a full tablet report, sending...
00:13:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.409821 29248 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.411289 29127 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:54.419 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786493605642, tablet=577804064edd41e4802dc27206597d97, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:13:54.426 [DEBUG - main] (Connection.java:575) [peer master-127.28.97.60:44689(127.28.97.60:44689)] disconnecting while in state READY
00:13:54.427 [DEBUG - main] (Connection.java:575) [peer master-127.28.97.62:40027(127.28.97.62:40027)] disconnecting while in state READY
00:13:54.428 [DEBUG - main] (Connection.java:575) [peer f962932df48b42568f69e73e7e4fd476(127.28.97.1:41751)] disconnecting while in state TERMINATED
00:13:54.428 [DEBUG - main] (Connection.java:575) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] disconnecting while in state READY
00:13:54.428 [DEBUG - main] (Connection.java:575) [peer master-127.28.97.61:39769(127.28.97.61:39769)] disconnecting while in state READY
00:13:54.428 [DEBUG - main] (Connection.java:575) [peer e9fbd638be254004a6d0df9c49d25e3a(127.28.97.2:35557)] disconnecting while in state TERMINATED
00:13:54.431 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.28.97.60:44689(127.28.97.60:44689)] handling channelInactive
00:13:54.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.28.97.60:44689(127.28.97.60:44689)] cleaning up while in state READY due to: connection closed
00:13:54.432 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.97.61:39769(127.28.97.61:39769)] handling channelInactive
00:13:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.430186   605 heartbeater.cc:344] Connected to a master server at 127.28.97.60:44689
00:13:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.430330   605 heartbeater.cc:461] Registering TS with master...
00:13:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.430622   605 heartbeater.cc:507] Master 127.28.97.60:44689 requested a full tablet report, sending...
00:13:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.431762 29353 ts_manager.cc:194] Re-registered known tserver with Master: e9fbd638be254004a6d0df9c49d25e3a (127.28.97.2:35557)
00:13:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.432821 29353 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.97.2:35343
00:13:54.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.97.61:39769(127.28.97.61:39769)] cleaning up while in state READY due to: connection closed
00:13:54.436 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:13:54.438 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.97.62:40027(127.28.97.62:40027)] handling channelInactive
00:13:54.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.97.62:40027(127.28.97.62:40027)] cleaning up while in state READY due to: connection closed
00:13:54.439 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] handling channelInactive
00:13:54.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8f51a58ed1ee47e1ba1b670c49abba2b(127.28.97.3:44737)] cleaning up while in state READY due to: connection closed
00:13:54.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.440979 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 32589
00:13:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.464089   422 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:13:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.464686   422 tablet_bootstrap.cc:492] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Bootstrap complete.
00:13:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.466230   422 ts_tablet_manager.cc:1403] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a: Time spent bootstrapping tablet: real 0.095s	user 0.052s	sys 0.010s
00:13:54.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.469169 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 391
00:13:54.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.468766   422 raft_consensus.cc:359] T 577804064edd41e4802dc27206597d97 P e9fbd638be254004a6d0df9c49d25e3a [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f51a58ed1ee47e1ba1b670c49abba2b" member_type: VOTER last_known_addr { host: "127.28.97.3" port: 44737 } } peers { permanent_uuid: "e9fbd638be254004a6d0df9c49d25e3a" member_type: VOTER last_known_addr { host: "127.28.97.2" port: 35557 } } peers { permanent_uuid: "f962932df48b42568f69e73e7e4fd476" member_type: VOTER last_known_addr { host: "127.28.97.1" port: 41751 } }
00:13:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.486318 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 32297
00:13:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.502487 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 29082
00:13:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.519590 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 29196
00:13:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 00:13:54.536505 29060 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskC8xxxa/build/debug/bin/kudu with pid 29300
00:13:54.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T00:13:54Z chronyd exiting

Time: 31.854

OK (1 test)

