JUnit version 4.13.2
.18:21:10.753 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783534870744
18:21:10.817 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
18:21:10.852 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
18:21:10.866 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task93oYzr/test-tmp
18:21:10.893 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
18:21:10.897 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task93oYzr/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
18:21:11.106 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

18:21:11.120 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:21:11.121 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

18:21:11.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.122385 22594 mini_oidc.cc:150] Starting JWKS server
18:21:11.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.123152 22594 webserver.cc:534] Webserver started at http://127.0.0.1:33423/ using document root <none> and password file <none>
18:21:11.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.123240 22594 mini_oidc.cc:186] Starting OIDC Discovery server
18:21:11.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.123435 22594 webserver.cc:534] Webserver started at http://127.0.0.1:38053/ using document root <none> and password file <none>
18:21:11.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T18:21:11Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
18:21:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T18:21:11Z Disabled control of system clock
18:21:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.136657 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/wal
18:21:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data
18:21:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/logs
18:21:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data/info.pb
18:21:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:21:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:21:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:21:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.190:35499
18:21:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.190
18:21:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:21:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:21:11.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33423/jwks.json with env {}
18:21:11.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.215737 22605 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:11.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.215946 22605 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.215978 22605 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.218065 22605 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:21:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.218108 22605 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:11.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.218127 22605 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:21:11.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.218144 22605 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:21:11.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.218161 22605 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:21:11.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.218178 22605 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33423/jwks.json
18:21:11.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.220479 22605 master_runner.cc:387] Master server non-default flags:
18:21:11.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data
18:21:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/wal
18:21:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.190:35499
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:21:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33423/jwks.json
18:21:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data/info.pb
18:21:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.190
18:21:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:11.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.22605
18:21:11.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:11.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:11.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/logs
18:21:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:21:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.220705 22605 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.220980 22605 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:11.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.223889 22611 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.223930 22605 server_base.cc:1061] running on GCE node
18:21:11.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.224041 22610 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.224187 22613 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.224422 22605 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.224681 22605 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.225842 22605 hybrid_clock.cc:648] HybridClock initialized: now 1783534871225819 us; error 31 us; skew 500 ppm
18:21:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.227108 22605 webserver.cc:534] Webserver started at http://127.22.16.190:33667/ using document root <none> and password file <none>
18:21:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.227339 22605 fs_manager.cc:362] Metadata directory not provided
18:21:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.227397 22605 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.227495 22605 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:21:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.228436 22605 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data/instance:
18:21:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20237fa554614b2f95ee8f6b0f811e5a"
18:21:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.228780 22605 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/wal/instance:
18:21:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20237fa554614b2f95ee8f6b0f811e5a"
18:21:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.230489 22605 fs_manager.cc:696] Time spent creating directory manager: real 0.002s	user 0.000s	sys 0.003s
18:21:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.231137 22619 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.231324 22605 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.001s
18:21:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.231379 22605 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/wal
18:21:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20237fa554614b2f95ee8f6b0f811e5a"
18:21:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.231421 22605 fs_report.cc:389] FS layout report
18:21:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/wal
18:21:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/wal
18:21:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data/data
18:21:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:11.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.239796 22605 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.240025 22605 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.240146 22605 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.244133 22605 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.190:35499
18:21:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.244165 22671 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.190:35499 every 8 connection(s)
18:21:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.244407 22605 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/data/info.pb
18:21:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.244910 22672 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:21:11.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.246098 22672 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.250824 22672 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.251238 22623 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.189:39739: connect: Connection refused (error 111)
18:21:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.251567 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 22605
18:21:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.251675 22594 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-0/wal/instance
18:21:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.251998 22672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.16.189:39739: Network error: Client connection negotiation failed: client connection to 127.22.16.189:39739: connect: Connection refused (error 111)
18:21:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.252478 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/wal
18:21:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data
18:21:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/logs
18:21:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data/info.pb
18:21:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.189:39739
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.189
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:21:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:21:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33423/jwks.json with env {}
18:21:11.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.301203 22672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } attempt: 1
18:21:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.302145 22672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.16.189:39739: Network error: Client connection negotiation failed: client connection to 127.22.16.189:39739: connect: Connection refused (error 111)
18:21:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.328037 22677 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.328186 22677 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.328204 22677 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.329679 22677 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:21:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.329720 22677 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.329733 22677 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:21:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.329746 22677 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:21:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.329757 22677 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:21:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.329768 22677 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33423/jwks.json
18:21:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.331861 22677 master_runner.cc:387] Master server non-default flags:
18:21:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data
18:21:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/wal
18:21:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:21:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:21:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.189:39739
18:21:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:21:11.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33423/jwks.json
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data/info.pb
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.189
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.22677
18:21:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/logs
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.332118 22677 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.332340 22677 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.335359 22685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.335439 22683 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.335572 22682 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.336071 22677 server_base.cc:1061] running on GCE node
18:21:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.336292 22677 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.336575 22677 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.337740 22677 hybrid_clock.cc:648] HybridClock initialized: now 1783534871337696 us; error 56 us; skew 500 ppm
18:21:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.339082 22677 webserver.cc:534] Webserver started at http://127.22.16.189:37505/ using document root <none> and password file <none>
18:21:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.339308 22677 fs_manager.cc:362] Metadata directory not provided
18:21:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.339368 22677 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.339473 22677 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:21:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.340515 22677 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data/instance:
18:21:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "611763c48c0646bebb521377b06adf31"
18:21:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.340876 22677 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/wal/instance:
18:21:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "611763c48c0646bebb521377b06adf31"
18:21:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.342417 22677 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.003s	sys 0.000s
18:21:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.343273 22691 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.343441 22677 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.343506 22677 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/wal
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "611763c48c0646bebb521377b06adf31"
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.343562 22677 fs_report.cc:389] FS layout report
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/wal
18:21:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/wal
18:21:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data/data
18:21:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.355957 22677 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.356287 22677 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.356413 22677 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:11.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.360561 22677 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.189:39739
18:21:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.360606 22743 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.189:39739 every 8 connection(s)
18:21:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.360976 22677 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/data/info.pb
18:21:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.361670 22744 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:21:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.363355 22744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.366879 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 22677
18:21:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.366969 22594 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-1/wal/instance
18:21:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.367390 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/wal
18:21:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data
18:21:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/logs
18:21:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data/info.pb
18:21:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:21:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:21:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:21:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.188:46685
18:21:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.188
18:21:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:21:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:21:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33423/jwks.json with env {}
18:21:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.370285 22672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } attempt: 2
18:21:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.370437 22744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.375160 22672 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.375801 22672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.16.188:46685: Network error: Client connection negotiation failed: client connection to 127.22.16.188:46685: connect: Connection refused (error 111)
18:21:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.375783 22744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.376237 22694 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.188:46685: connect: Connection refused (error 111)
18:21:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.376771 22744 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.16.188:46685: Network error: Client connection negotiation failed: client connection to 127.22.16.188:46685: connect: Connection refused (error 111)
18:21:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.418984 22672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } attempt: 1
18:21:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.419963 22672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.16.188:46685: Network error: Client connection negotiation failed: client connection to 127.22.16.188:46685: connect: Connection refused (error 111)
18:21:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.425951 22744 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } attempt: 1
18:21:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.426731 22744 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.16.188:46685: Network error: Client connection negotiation failed: client connection to 127.22.16.188:46685: connect: Connection refused (error 111)
18:21:11.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.448885 22747 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:11.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.449052 22747 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:11.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.449071 22747 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.450989 22747 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:21:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.451049 22747 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.451069 22747 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:21:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.451086 22747 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:21:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.451104 22747 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:21:11.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.451119 22747 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33423/jwks.json
18:21:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.453253 22747 master_runner.cc:387] Master server non-default flags:
18:21:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/wal
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:21:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.188:46685
18:21:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:21:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:21:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33423/jwks.json
18:21:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data/info.pb
18:21:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.188
18:21:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.22747
18:21:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/logs
18:21:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:21:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.453466 22747 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.453744 22747 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.455926 22758 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.456048 22755 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.456120 22756 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.456352 22747 server_base.cc:1061] running on GCE node
18:21:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.456544 22747 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.456748 22747 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.460026 22747 hybrid_clock.cc:648] HybridClock initialized: now 1783534871459998 us; error 35 us; skew 500 ppm
18:21:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.461376 22747 webserver.cc:534] Webserver started at http://127.22.16.188:40909/ using document root <none> and password file <none>
18:21:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.461613 22747 fs_manager.cc:362] Metadata directory not provided
18:21:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.461673 22747 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.461779 22747 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:21:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.462783 22747 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data/instance:
18:21:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfda65c815434c42b529c3b951730c00"
18:21:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.463140 22747 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/wal/instance:
18:21:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfda65c815434c42b529c3b951730c00"
18:21:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.464728 22747 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.001s
18:21:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.465663 22764 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.465883 22747 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:21:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.465955 22747 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/wal
18:21:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfda65c815434c42b529c3b951730c00"
18:21:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.466013 22747 fs_report.cc:389] FS layout report
18:21:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/wal
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/wal
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data/data
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.467095 22672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } attempt: 2
18:21:11.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.467748 22672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.16.188:46685: Network error: Client connection negotiation failed: client connection to 127.22.16.188:46685: connect: Connection refused (error 111)
18:21:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.488121 22747 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.488523 22747 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:11.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.488658 22747 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.493203 22747 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.188:46685
18:21:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.493235 22816 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.188:46685 every 8 connection(s)
18:21:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.493676 22747 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/data/info.pb
18:21:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.494261 22817 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:21:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.494908 22744 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } attempt: 2
18:21:11.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.497635 22817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.501786 22817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.501845 22744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: Bootstrap starting.
18:21:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.502557 22744 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: Neither blocks nor log segments found. Creating new log.
18:21:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.502825 22744 log.cc:826] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: Log is configured to *not* fsync() on all Append() calls
18:21:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.503295 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 22747
18:21:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.503369 22594 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/master-2/wal/instance
18:21:11.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.504967 22817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } has no permanent_uuid. Determining permanent_uuid...
18:21:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.505400 22744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: No bootstrap required, opened a new log
18:21:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.506860 22744 raft_consensus.cc:359] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.506934 22744 raft_consensus.cc:385] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:21:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.506953 22744 raft_consensus.cc:740] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 611763c48c0646bebb521377b06adf31, State: Initialized, Role: FOLLOWER
18:21:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.507025 22744 consensus_queue.cc:260] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.507221 22744 sys_catalog.cc:565] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [sys.catalog]: configured and running, proceeding with master startup.
18:21:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.507251 22823 sys_catalog.cc:455] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.507341 22823 sys_catalog.cc:458] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [sys.catalog]: This master's current role is: FOLLOWER
18:21:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.511214 22817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00: Bootstrap starting.
18:21:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.509690 22677 master_runner.cc:187] Error getting master registration for 127.22.16.190:35499: OK, instance_id { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" instance_seqno: 1783534871242988 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:21:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.511420 22677 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
18:21:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.511519 22835 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:21:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.511567 22835 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:21:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.511953 22817 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00: Neither blocks nor log segments found. Creating new log.
18:21:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.512193 22817 log.cc:826] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00: Log is configured to *not* fsync() on all Append() calls
18:21:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.513309 22817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00: No bootstrap required, opened a new log
18:21:11.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.515239 22817 raft_consensus.cc:359] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.515394 22817 raft_consensus.cc:385] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:21:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.515432 22817 raft_consensus.cc:740] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfda65c815434c42b529c3b951730c00, State: Initialized, Role: FOLLOWER
18:21:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.515551 22817 consensus_queue.cc:260] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.515830 22817 sys_catalog.cc:565] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: configured and running, proceeding with master startup.
18:21:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.518806 22747 master_runner.cc:187] Error getting master registration for 127.22.16.190:35499: OK, instance_id { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" instance_seqno: 1783534871242988 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:21:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.520466 22747 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
18:21:11.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.517689 22838 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.523016 22838 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: This master's current role is: FOLLOWER
18:21:11.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.523667 22849 catalog_manager.cc:1593] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:21:11.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.523723 22849 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:21:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.527981 22838 raft_consensus.cc:493] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:21:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.528074 22838 raft_consensus.cc:515] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.528283 22838 leader_election.cc:290] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 20237fa554614b2f95ee8f6b0f811e5a (127.22.16.190:35499), 611763c48c0646bebb521377b06adf31 (127.22.16.189:39739)
18:21:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.528604 22718 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dfda65c815434c42b529c3b951730c00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "611763c48c0646bebb521377b06adf31" is_pre_election: true
18:21:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.528716 22718 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dfda65c815434c42b529c3b951730c00 in term 0.
18:21:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.529093 22768 leader_election.cc:304] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 611763c48c0646bebb521377b06adf31, dfda65c815434c42b529c3b951730c00; no voters: 
18:21:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.529210 22838 raft_consensus.cc:2804] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Leader pre-election won for term 1
18:21:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.529259 22838 raft_consensus.cc:493] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:21:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.529284 22838 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 0 FOLLOWER]: Advancing to term 1
18:21:11.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.530124 22838 raft_consensus.cc:515] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.530241 22838 leader_election.cc:290] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [CANDIDATE]: Term 1 election: Requested vote from peers 20237fa554614b2f95ee8f6b0f811e5a (127.22.16.190:35499), 611763c48c0646bebb521377b06adf31 (127.22.16.189:39739)
18:21:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.530439 22718 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dfda65c815434c42b529c3b951730c00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "611763c48c0646bebb521377b06adf31"
18:21:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.530503 22718 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [term 0 FOLLOWER]: Advancing to term 1
18:21:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.531302 22718 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dfda65c815434c42b529c3b951730c00 in term 1.
18:21:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.531450 22768 leader_election.cc:304] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 611763c48c0646bebb521377b06adf31, dfda65c815434c42b529c3b951730c00; no voters: 
18:21:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.531520 22838 raft_consensus.cc:2804] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 1 FOLLOWER]: Leader election won for term 1
18:21:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.531656 22838 raft_consensus.cc:697] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [term 1 LEADER]: Becoming Leader. State: Replica: dfda65c815434c42b529c3b951730c00, State: Running, Role: LEADER
18:21:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.531729 22838 consensus_queue.cc:237] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.532080 22838 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: SysCatalogTable state changed. Reason: New leader dfda65c815434c42b529c3b951730c00. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.532140 22838 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: This master's current role is: LEADER
18:21:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.532351 22853 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
18:21:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.532523 22853 catalog_manager.cc:1519] Initializing Kudu cluster ID...
18:21:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.534119 22718 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [term 1 FOLLOWER]: Refusing update from remote peer dfda65c815434c42b529c3b951730c00: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:21:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.534346 22838 consensus_queue.cc:1048] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:21:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.535410 22823 sys_catalog.cc:455] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [sys.catalog]: SysCatalogTable state changed. Reason: New leader dfda65c815434c42b529c3b951730c00. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.535481 22823 sys_catalog.cc:458] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [sys.catalog]: This master's current role is: FOLLOWER
18:21:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.548837 22854 mvcc.cc:204] Tried to move back new op lower bound from 7305358833801814016 to 7305358833794580480. Current Snapshot: MvccSnapshot[applied={T|T < 7305358833801814016}]
18:21:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.550256 22838 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.550344 22838 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: This master's current role is: LEADER
18:21:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.550962 22853 catalog_manager.cc:1382] Generated new cluster ID: 24e1f130b87c4dcb8fad804d5194aac7
18:21:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.551019 22853 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
18:21:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.551800 22823 sys_catalog.cc:455] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.551877 22823 sys_catalog.cc:458] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31 [sys.catalog]: This master's current role is: FOLLOWER
18:21:11.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.555300 22855 mvcc.cc:204] Tried to move back new op lower bound from 7305358833801814016 to 7305358833794580480. Current Snapshot: MvccSnapshot[applied={T|T < 7305358833801814016}]
18:21:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.574875 22672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } attempt: 3
18:21:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.580144 22672 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: Bootstrap starting.
18:21:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.580614 22853 catalog_manager.cc:1405] Generated new certificate authority record
18:21:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.580845 22672 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: Neither blocks nor log segments found. Creating new log.
18:21:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.581085 22672 log.cc:826] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: Log is configured to *not* fsync() on all Append() calls
18:21:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.582079 22853 catalog_manager.cc:1539] Loading token signing keys...
18:21:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.582396 22672 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: No bootstrap required, opened a new log
18:21:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.584200 22672 raft_consensus.cc:359] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.584316 22672 raft_consensus.cc:385] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:21:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.584348 22672 raft_consensus.cc:740] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20237fa554614b2f95ee8f6b0f811e5a, State: Initialized, Role: FOLLOWER
18:21:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.584436 22672 consensus_queue.cc:260] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } }
18:21:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.584693 22672 sys_catalog.cc:565] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [sys.catalog]: configured and running, proceeding with master startup.
18:21:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.584740 22645 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dfda65c815434c42b529c3b951730c00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20237fa554614b2f95ee8f6b0f811e5a"
18:21:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.584859 22645 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [term 0 FOLLOWER]: Advancing to term 1
18:21:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.585628 22645 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dfda65c815434c42b529c3b951730c00 in term 1.
18:21:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.585783 22644 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [term 1 FOLLOWER]: Refusing update from remote peer dfda65c815434c42b529c3b951730c00: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:21:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.585894 22646 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dfda65c815434c42b529c3b951730c00" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20237fa554614b2f95ee8f6b0f811e5a" is_pre_election: true
18:21:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.585949 22646 raft_consensus.cc:2376] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate dfda65c815434c42b529c3b951730c00 in term 1. Re-sending same reply.
18:21:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.589252 22851 consensus_queue.cc:1048] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:21:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.593320 22872 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:21:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.593706 22872 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:21:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.594015 22873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.594316 22873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [sys.catalog]: This master's current role is: FOLLOWER
18:21:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.594872 22873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [sys.catalog]: SysCatalogTable state changed. Reason: New leader dfda65c815434c42b529c3b951730c00. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.595216 22873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [sys.catalog]: This master's current role is: FOLLOWER
18:21:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.595300 22873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.595346 22873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a [sys.catalog]: This master's current role is: FOLLOWER
18:21:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.597188 22874 mvcc.cc:204] Tried to move back new op lower bound from 7305358833983590400 to 7305358833794580480. Current Snapshot: MvccSnapshot[applied={T|T < 7305358833801814016}]
18:21:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.605754 22851 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "dfda65c815434c42b529c3b951730c00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20237fa554614b2f95ee8f6b0f811e5a" member_type: VOTER last_known_addr { host: "127.22.16.190" port: 35499 } } peers { permanent_uuid: "611763c48c0646bebb521377b06adf31" member_type: VOTER last_known_addr { host: "127.22.16.189" port: 39739 } } peers { permanent_uuid: "dfda65c815434c42b529c3b951730c00" member_type: VOTER last_known_addr { host: "127.22.16.188" port: 46685 } } }
18:21:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.605859 22851 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00 [sys.catalog]: This master's current role is: LEADER
18:21:11.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.607335 22853 catalog_manager.cc:6069] T 00000000000000000000000000000000 P dfda65c815434c42b529c3b951730c00: Generated new TSK 0
18:21:11.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.607769 22853 catalog_manager.cc:1549] Initializing in-progress tserver states...
18:21:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.616420 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:0
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:11.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.731617 22880 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:11.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.732344 22880 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:11.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.732671 22880 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.735318 22880 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.735637 22880 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.129
18:21:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.738232 22880 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:0
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.22880
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.742020 22880 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.742529 22880 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:11.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.745704 22888 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.746620 22886 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.755525 22885 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.764293 22880 server_base.cc:1061] running on GCE node
18:21:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.764482 22880 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.764698 22880 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.780920 22880 hybrid_clock.cc:648] HybridClock initialized: now 1783534871780895 us; error 52 us; skew 500 ppm
18:21:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.782749 22880 webserver.cc:534] Webserver started at http://127.22.16.129:44715/ using document root <none> and password file <none>
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.782960 22880 fs_manager.cc:362] Metadata directory not provided
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.783010 22880 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.783107 22880 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.784240 22880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/instance:
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f6251917bba4996b6e74abedae69139"
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.784590 22880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal/instance:
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f6251917bba4996b6e74abedae69139"
18:21:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.787140 22880 fs_manager.cc:696] Time spent creating directory manager: real 0.002s	user 0.000s	sys 0.002s
18:21:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.788255 22894 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.788424 22880 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:21:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.788478 22880 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f6251917bba4996b6e74abedae69139"
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.788532 22880 fs_report.cc:389] FS layout report
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/data
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.836745 22880 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.837105 22880 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.837227 22880 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.837422 22880 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.837802 22880 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:21:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.837842 22880 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:21:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.837873 22880 ts_tablet_manager.cc:616] Registered 0 tablets
18:21:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.837894 22880 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:21:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.845417 22880 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.129:41735
18:21:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.845456 23007 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.129:41735 every 8 connection(s)
18:21:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.845854 22880 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.847725 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 22880
18:21:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.847823 22594 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal/instance
18:21:11.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.850054 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:0
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852293 23009 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852263 23008 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852415 23009 heartbeater.cc:461] Registering TS with master...
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852415 23008 heartbeater.cc:461] Registering TS with master...
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852644 23008 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852275 23010 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852756 23010 heartbeater.cc:461] Registering TS with master...
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852854 23010 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.852649 23009 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.853160 22636 ts_manager.cc:194] Registered new tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.853214 22708 ts_manager.cc:194] Registered new tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.853530 22781 ts_manager.cc:194] Registered new tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.854171 22781 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.129:47219
18:21:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.930823 23015 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.931015 23015 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.931048 23015 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.932763 23015 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.932857 23015 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.130
18:21:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.934502 23015 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:0
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23015
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.934782 23015 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.935056 23015 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.938019 23021 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.938076 23023 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.938081 23015 server_base.cc:1061] running on GCE node
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:11.938232 23020 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.938406 23015 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.938645 23015 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:11.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.939839 23015 hybrid_clock.cc:648] HybridClock initialized: now 1783534871939804 us; error 44 us; skew 500 ppm
18:21:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.940997 23015 webserver.cc:534] Webserver started at http://127.22.16.130:33841/ using document root <none> and password file <none>
18:21:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.941188 23015 fs_manager.cc:362] Metadata directory not provided
18:21:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.941222 23015 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.941293 23015 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:21:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.942052 23015 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/instance:
18:21:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.942304 23015 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal/instance:
18:21:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.943375 23015 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.001s	sys 0.001s
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.944026 23029 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.944211 23015 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.944274 23015 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.944332 23015 fs_report.cc:389] FS layout report
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:11.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/data
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.955700 23015 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.956020 23015 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.956143 23015 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.956373 23015 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.956753 23015 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:21:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.956815 23015 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:21:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.956846 23015 ts_tablet_manager.cc:616] Registered 0 tablets
18:21:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.956863 23015 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:21:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.963007 23015 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.130:37215
18:21:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.963100 23142 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.130:37215 every 8 connection(s)
18:21:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.963423 23015 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.966758 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 23015
18:21:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.966849 22594 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal/instance
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.968434 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:0
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.969352 23144 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.969439 23144 heartbeater.cc:461] Registering TS with master...
18:21:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.969584 23144 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.969882 23145 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.969938 23145 heartbeater.cc:461] Registering TS with master...
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.969929 22781 ts_manager.cc:194] Registered new tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.970036 23145 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.970244 23143 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.970305 23143 heartbeater.cc:461] Registering TS with master...
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.970391 23143 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.970433 22781 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.130:40603
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.970638 22708 ts_manager.cc:194] Registered new tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:11.970814 22636 ts_manager.cc:194] Registered new tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.045751 23150 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.045955 23150 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.045986 23150 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:12.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.047678 23150 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.047772 23150 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.131
18:21:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.049664 23150 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:0
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:21:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23150
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.049942 23150 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.050216 23150 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.053004 23156 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.052996 23155 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.053448 23158 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.053644 23150 server_base.cc:1061] running on GCE node
18:21:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.053822 23150 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:12.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.053997 23150 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.055117 23150 hybrid_clock.cc:648] HybridClock initialized: now 1783534872055088 us; error 47 us; skew 500 ppm
18:21:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.056190 23150 webserver.cc:534] Webserver started at http://127.22.16.131:41001/ using document root <none> and password file <none>
18:21:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.056344 23150 fs_manager.cc:362] Metadata directory not provided
18:21:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.056375 23150 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.056445 23150 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:21:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.057228 23150 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/instance:
18:21:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:12 on dist-test-slave-v72b"
18:21:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.057488 23150 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal/instance:
18:21:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:12 on dist-test-slave-v72b"
18:21:12.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.058555 23150 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.002s	sys 0.000s
18:21:12.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.059145 23164 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:12.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.059347 23150 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
18:21:12.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.059413 23150 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:12 on dist-test-slave-v72b"
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.059476 23150 fs_report.cc:389] FS layout report
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/data
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.070591 23150 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.070906 23150 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.071059 23150 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.071329 23150 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.071712 23150 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:21:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.071789 23150 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:21:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.071823 23150 ts_tablet_manager.cc:616] Registered 0 tablets
18:21:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.071841 23150 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:21:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.077522 23150 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.131:42931
18:21:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.077597 23277 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.131:42931 every 8 connection(s)
18:21:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.077919 23150 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.082861 23278 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.082864 23280 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.082983 23280 heartbeater.cc:461] Registering TS with master...
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083009 23279 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.082984 23278 heartbeater.cc:461] Registering TS with master...
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083060 23279 heartbeater.cc:461] Registering TS with master...
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083201 23278 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083201 23280 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083361 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 23150
18:21:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083431 22594 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal/instance
18:21:12.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083984 22636 ts_manager.cc:194] Registered new tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:12.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.083201 23279 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:12.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.084290 22708 ts_manager.cc:194] Registered new tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.084441 22781 ts_manager.cc:194] Registered new tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.084834 22781 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.131:56817
18:21:12.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.086798 22594 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
18:21:12.087 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:21:12.088 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

18:21:12.097 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.22.16.190"
      port: 35499
    }
    bound_http_address {
      host: "127.22.16.190"
      port: 33667
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.22.16.189"
      port: 39739
    }
    bound_http_address {
      host: "127.22.16.189"
      port: 37505
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.22.16.188"
      port: 46685
    }
    bound_http_address {
      host: "127.22.16.188"
      port: 40909
    }
  }
}

18:21:12.100 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

18:21:12.102 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.22.16.129"
      port: 41735
    }
    bound_http_address {
      host: "127.22.16.129"
      port: 44715
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.22.16.130"
      port: 37215
    }
    bound_http_address {
      host: "127.22.16.130"
      port: 33841
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.22.16.131"
      port: 42931
    }
    bound_http_address {
      host: "127.22.16.131"
      port: 41001
    }
  }
}

18:21:12.102 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
18:21:12.279 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
18:21:12.334 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
18:21:12.482 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.16.190:35499(127.22.16.190:35499)] connecting to peer
18:21:12.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.512008 22835 catalog_manager.cc:1294] Loaded cluster ID: 24e1f130b87c4dcb8fad804d5194aac7
18:21:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.512082 22835 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: loading cluster ID for follower catalog manager: success
18:21:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.513437 22835 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: acquiring CA information for follower catalog manager: success
18:21:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.513760 22835 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 611763c48c0646bebb521377b06adf31: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:21:12.531 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.16.188:46685(127.22.16.188:46685)] connecting to peer
18:21:12.533 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.16.189:39739(127.22.16.189:39739)] connecting to peer
18:21:12.551 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.16.188:46685(127.22.16.188:46685)] Successfully connected to peer
18:21:12.551 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.22.16.189:39739(127.22.16.189:39739)] Successfully connected to peer
18:21:12.551 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.16.190:35499(127.22.16.190:35499)] Successfully connected to peer
18:21:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.594375 22872 catalog_manager.cc:1294] Loaded cluster ID: 24e1f130b87c4dcb8fad804d5194aac7
18:21:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.594451 22872 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: loading cluster ID for follower catalog manager: success
18:21:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.595788 22872 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: acquiring CA information for follower catalog manager: success
18:21:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.596129 22872 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 20237fa554614b2f95ee8f6b0f811e5a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:21:12.653 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.16.189
18:21:12.653 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.16.188
18:21:12.653 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.16.190
18:21:12.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.854764 23009 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:12.866 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1259d113, L:/127.0.0.1:42846 - R:/127.22.16.190:35499] using SASL/PLAIN
18:21:12.866 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x02ad97cc, L:/127.0.0.1:42852 - R:/127.22.16.188:46685] using SASL/PLAIN
18:21:12.867 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x9d00be4d, L:/127.0.0.1:47798 - R:/127.22.16.189:39739] using SASL/PLAIN
18:21:12.902 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
18:21:12.903 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=459), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.16.190:35499, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.16.188:46685, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.16.189:39739, [433ms] Sub RPC ConnectToMaster: received response from server master-127.22.16.190:35499: OK, [434ms] Sub RPC ConnectToMaster: received response from server master-127.22.16.189:39739: OK, [434ms] Sub RPC ConnectToMaster: received response from server master-127.22.16.188:46685: OK, deferred=null) after lookup
18:21:12.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.950019 22781 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42852:
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432"
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:21:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:21:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:21:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:21:12.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:21:12.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
18:21:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
18:21:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
18:21:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
18:21:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
18:21:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:21:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:21:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:12.950399 22781 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
18:21:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.966192 22942 tablet_service.cc:1511] Processing CreateTablet for tablet f98f377903864cbda11858a1e6d986be (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432 [id=3e11d92c0f7e472baa4724eb6f8390dc]), partition=RANGE (key) PARTITION UNBOUNDED
18:21:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.966266 23077 tablet_service.cc:1511] Processing CreateTablet for tablet f98f377903864cbda11858a1e6d986be (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432 [id=3e11d92c0f7e472baa4724eb6f8390dc]), partition=RANGE (key) PARTITION UNBOUNDED
18:21:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.966499 22942 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f98f377903864cbda11858a1e6d986be. 1 dirs total, 0 dirs full, 0 dirs failed
18:21:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.966526 23077 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f98f377903864cbda11858a1e6d986be. 1 dirs total, 0 dirs full, 0 dirs failed
18:21:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.968801 23307 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap starting.
18:21:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.969410 23307 tablet_bootstrap.cc:654] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Neither blocks nor log segments found. Creating new log.
18:21:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.969717 23307 log.cc:826] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Log is configured to *not* fsync() on all Append() calls
18:21:12.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.970250 23308 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap starting.
18:21:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.970384 23212 tablet_service.cc:1511] Processing CreateTablet for tablet f98f377903864cbda11858a1e6d986be (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432 [id=3e11d92c0f7e472baa4724eb6f8390dc]), partition=RANGE (key) PARTITION UNBOUNDED
18:21:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.970655 23212 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f98f377903864cbda11858a1e6d986be. 1 dirs total, 0 dirs full, 0 dirs failed
18:21:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.971138 23308 tablet_bootstrap.cc:654] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Neither blocks nor log segments found. Creating new log.
18:21:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.971817 23144 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.972769 23308 log.cc:826] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Log is configured to *not* fsync() on all Append() calls
18:21:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.973667 23307 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: No bootstrap required, opened a new log
18:21:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.973729 23307 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent bootstrapping tablet: real 0.005s	user 0.001s	sys 0.000s
18:21:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.975318 23307 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.976115 23307 raft_consensus.cc:385] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:21:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.976343 23307 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f6251917bba4996b6e74abedae69139, State: Initialized, Role: FOLLOWER
18:21:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.976619 23307 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.977046 23307 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent starting tablet: real 0.003s	user 0.002s	sys 0.000s
18:21:12.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.979435 23311 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap starting.
18:21:12.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.980233 23311 tablet_bootstrap.cc:654] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Neither blocks nor log segments found. Creating new log.
18:21:12.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.980685 23311 log.cc:826] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Log is configured to *not* fsync() on all Append() calls
18:21:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.981428 23308 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: No bootstrap required, opened a new log
18:21:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.981508 23308 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent bootstrapping tablet: real 0.011s	user 0.002s	sys 0.000s
18:21:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.983317 23308 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.983449 23308 raft_consensus.cc:385] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:21:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.983489 23308 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Initialized, Role: FOLLOWER
18:21:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.983580 23308 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.983771 23308 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.002s
18:21:12.986 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 3e11d92c0f7e472baa4724eb6f8390dc
18:21:12.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.987838 23311 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: No bootstrap required, opened a new log
18:21:12.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.988106 23311 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent bootstrapping tablet: real 0.009s	user 0.002s	sys 0.000s
18:21:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.989848 23311 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.989995 23311 raft_consensus.cc:385] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:21:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.990025 23311 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Initialized, Role: FOLLOWER
18:21:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.990149 23311 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.990374 23311 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
18:21:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:12.990651 23279 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:13.079038 23281 tablet.cc:2406] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:21:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:13.097293 23011 tablet.cc:2406] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:21:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.203840 23315 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:21:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.203994 23315 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.204339 23315 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.208467 22962 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139" is_pre_election: true
18:21:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.208630 22962 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 0.
18:21:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.208734 23097 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2" is_pre_election: true
18:21:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.208833 23165 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 
18:21:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.208880 23097 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 0.
18:21:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.208950 23315 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Leader pre-election won for term 1
18:21:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.208986 23315 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:21:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.209007 23315 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 0 FOLLOWER]: Advancing to term 1
18:21:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.209817 23315 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.209954 23315 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 1 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.210232 23097 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.210278 22962 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139"
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.210319 23097 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 0 FOLLOWER]: Advancing to term 1
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.210354 22962 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 0 FOLLOWER]: Advancing to term 1
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.211076 23097 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 1.
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.211133 22962 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 1.
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.211269 23165 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.211370 23315 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 1 FOLLOWER]: Leader election won for term 1
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.211494 23315 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 1 LEADER]: Becoming Leader. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Running, Role: LEADER
18:21:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.211571 23315 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.212540 22781 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131). New cstate: current_term: 1 leader_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: HEALTHY } } }
18:21:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:13.214591 23146 tablet.cc:2406] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:21:13.270 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 3e11d92c0f7e472baa4724eb6f8390dc
18:21:13.285 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:13.291 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:13.292 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [7ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=null) after lookup
18:21:13.301 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:13.304 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] Successfully connected to peer
18:21:13.380 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xa3a9c653, L:/127.0.0.1:49182 - R:/127.22.16.131:42931] using TOKEN/null
18:21:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.383204 23097 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Refusing update from remote peer 6f3eada3ec394bf9b1cc2e2a14c0e505: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:21:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.383502 22962 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Refusing update from remote peer 6f3eada3ec394bf9b1cc2e2a14c0e505: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:21:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.383772 23315 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:21:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.383889 23318 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:21:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.392504 23324 mvcc.cc:204] Tried to move back new op lower bound from 7305358841374941184 to 7305358840675057664. Current Snapshot: MvccSnapshot[applied={T|T < 7305358841374941184}]
18:21:13.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.393704 23322 mvcc.cc:204] Tried to move back new op lower bound from 7305358841374941184 to 7305358840675057664. Current Snapshot: MvccSnapshot[applied={T|T < 7305358841374941184 or (T in {7305358841374941184})}]
18:21:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.394097 23323 mvcc.cc:204] Tried to move back new op lower bound from 7305358841374941184 to 7305358840675057664. Current Snapshot: MvccSnapshot[applied={T|T < 7305358841374941184}]
18:21:13.461 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:13.462 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.131:42931
18:21:13.464 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:13.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:13.464359 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 23150
18:21:13.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:13.472 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] handling channelInactive
18:21:13.473 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state READY due to: connection closed
18:21:13.473 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: connection closed
18:21:13.487 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:13.487 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:13.488 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [3ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1258023431(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@7dbae9b8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6db1f1a9 -> wakeup thread Time-limited test)) after lookup
18:21:13.488 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:13.491 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.491 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.491 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.507 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:13.507 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:13.508 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [3ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [18ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1258023431(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@7dbae9b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6db1f1a9 -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101))) after lookup
18:21:13.509 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:13.511 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.511 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.511 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.549 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:13.550 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:13.550 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [3ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [18ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [38ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [41ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1258023431(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@7dbae9b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6db1f1a9 -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381))) after lookup
18:21:13.551 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:13.553 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.553 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.553 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.647 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:13.648 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:13.648 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=177), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [3ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [18ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [38ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [41ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [83ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [174ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [177ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1258023431(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@7dbae9b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6db1f1a9 -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931))) after lookup
18:21:13.649 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:13.651 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.651 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.651 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.748 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:13.748 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:13.749 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=278), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [3ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [18ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [38ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [41ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [83ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [174ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [177ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [179ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [181ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [181ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [277ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1258023431(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@7dbae9b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931) -> (continuation of Deferred@1126888884 after retry RPC@965815007), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6db1f1a9 -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931) -> (continuation of Deferred@1126888884 after retry RPC@965815007))) after lookup
18:21:13.750 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:13.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:13.752 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:14.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.915202 23336 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:14.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.915377 23336 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.915576 23337 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.915635 23337 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.915748 23336 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:14.916462 23032 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:14.916524 22897 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.916663 23337 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:14.917150 23032 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:14.917330 22897 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919269 22962 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139" is_pre_election: true
18:21:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919374 22962 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 1.
18:21:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919577 23030 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919700 23097 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2" is_pre_election: true
18:21:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919754 23336 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Leader pre-election won for term 2
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919812 23097 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f6251917bba4996b6e74abedae69139 in term 1.
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919835 23336 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919862 23336 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 1 FOLLOWER]: Advancing to term 2
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.919981 22895 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.920105 23337 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Leader pre-election won for term 2
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.920143 23337 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.920163 23337 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 1 FOLLOWER]: Advancing to term 2
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.920655 23336 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.920809 23336 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 2 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.920758 23337 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.920905 23337 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 2 election: Requested vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921010 22962 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139"
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921082 22962 raft_consensus.cc:2393] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 86740cf8056941e1bca1ebcce92945d2 in current term 2: Already voted for candidate 7f6251917bba4996b6e74abedae69139 in this term.
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921171 23097 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921260 23097 raft_consensus.cc:2393] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 7f6251917bba4996b6e74abedae69139 in current term 2: Already voted for candidate 86740cf8056941e1bca1ebcce92945d2 in this term.
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:14.921392 23032 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:14.921440 22897 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921543 22895 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f6251917bba4996b6e74abedae69139; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 86740cf8056941e1bca1ebcce92945d2
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921542 23030 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921640 23337 raft_consensus.cc:2749] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
18:21:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:14.921662 23336 raft_consensus.cc:2749] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
18:21:15.567 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:15.568 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:15.569 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2098), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [3ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [18ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [38ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [41ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [83ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [174ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [177ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [179ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [181ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [181ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [277ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [279ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [282ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [282ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2094ms] refreshing cache from master, [2094ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [2097ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1258023431(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@7dbae9b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931) -> (continuation of Deferred@1126888884 after retry RPC@965815007) -> (continuation of Deferred@1657698406 after retry RPC@1379492146), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6db1f1a9 -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931) -> (continuation of Deferred@1126888884 after retry RPC@965815007) -> (continuation of Deferred@1657698406 after retry RPC@1379492146))) after lookup
18:21:15.569 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:15.570 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:15.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:15.570 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:16.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.453529 23347 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.453617 23347 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.453776 23347 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.454042 22962 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139" is_pre_election: true
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.454141 22962 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 2.
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.454308 23030 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.454447 23347 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Leader pre-election won for term 3
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.454502 23347 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.454528 23347 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 2 FOLLOWER]: Advancing to term 3
18:21:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:16.454579 23032 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.455327 23347 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.455489 23347 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 3 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.455655 22962 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139"
18:21:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.455746 22962 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 2 FOLLOWER]: Advancing to term 3
18:21:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:16.455915 23032 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.456439 22962 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 3.
18:21:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.456629 23030 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.456779 23347 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 3 FOLLOWER]: Leader election won for term 3
18:21:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.456965 23347 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 3 LEADER]: Becoming Leader. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Running, Role: LEADER
18:21:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.457162 23347 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.458113 22781 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 reported cstate change: term changed from 1 to 3, leader changed from 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131) to 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130). New cstate: current_term: 3 leader_uuid: "86740cf8056941e1bca1ebcce92945d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: UNKNOWN } } }
18:21:16.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.879667 22962 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 3 FOLLOWER]: Refusing update from remote peer 86740cf8056941e1bca1ebcce92945d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
18:21:16.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:16.880002 23347 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
18:21:16.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:16.882038 23032 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 -> Peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Couldn't send request to peer 6f3eada3ec394bf9b1cc2e2a14c0e505. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:19.247 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:19.247 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:19.248 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=5777), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [3ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [17ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [18ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [37ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [38ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [41ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [83ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [83ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [174ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [177ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [179ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [181ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [181ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [275ms] refreshing cache from master, [275ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [277ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [279ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [282ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [282ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2094ms] refreshing cache from master, [2094ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [2097ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [2099ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [2100ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2100ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [5775ms] refreshing cache from master, [5775ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [5777ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1258023431(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@7dbae9b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931) -> (continuation of Deferred@1126888884 after retry RPC@965815007) -> (continuation of Deferred@1657698406 after retry RPC@1379492146) -> (continuation of Deferred@1767193746 after retry RPC@1733442674), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6db1f1a9 -> wakeup thread Time-limited test -> (continuation of Deferred@1172310341 after retry RPC@883299101) -> (continuation of Deferred@2011658238 after retry RPC@390347381) -> (continuation of Deferred@1421164484 after retry RPC@2033881931) -> (continuation of Deferred@1126888884 after retry RPC@965815007) -> (continuation of Deferred@1657698406 after retry RPC@1379492146) -> (continuation of Deferred@1767193746 after retry RPC@1733442674))) after lookup
18:21:19.249 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:19.250 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] Successfully connected to peer
18:21:19.265 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xdea720dc, L:/127.0.0.1:59488 - R:/127.22.16.130:37215] using TOKEN/null
18:21:19.274 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.131:42931
18:21:19.276 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.276578 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.353145 23383 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.353296 23383 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.353315 23383 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.354717 23383 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.354764 23383 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.131
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.356248 23383 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23383
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.356431 23383 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.356602 23383 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.359503 23388 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.359500 23391 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:19.359596 23389 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.359865 23383 server_base.cc:1061] running on GCE node
18:21:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.360020 23383 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.360234 23383 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.361377 23383 hybrid_clock.cc:648] HybridClock initialized: now 1783534879361361 us; error 29 us; skew 500 ppm
18:21:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.362618 23383 webserver.cc:534] Webserver started at http://127.22.16.131:41001/ using document root <none> and password file <none>
18:21:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.362812 23383 fs_manager.cc:362] Metadata directory not provided
18:21:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.362883 23383 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.364367 23383 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.365064 23397 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.365231 23383 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.365303 23383 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:12 on dist-test-slave-v72b"
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.365618 23383 fs_report.cc:389] FS layout report
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/data
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.384294 23383 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.384639 23383 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.384761 23383 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.385020 23383 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.385509 23404 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:19.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.386358 23383 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:19.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.386420 23383 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:19.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.386472 23383 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.387028 23383 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.387082 23383 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.387120 23404 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap starting.
18:21:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.393581 23383 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.131:42931
18:21:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.394033 23383 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:19.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.397332 23404 log.cc:826] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Log is configured to *not* fsync() on all Append() calls
18:21:19.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.399806 23404 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.399995 23511 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.131:42931 every 8 connection(s)
18:21:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.400133 23404 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap complete.
18:21:19.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.401273 23404 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent bootstrapping tablet: real 0.014s	user 0.004s	sys 0.007s
18:21:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.401934 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 23383
18:21:19.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:19.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.404237 23512 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:19.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.404328 23512 heartbeater.cc:461] Registering TS with master...
18:21:19.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.404508 23512 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:19.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.403123 23404 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:19.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.404767 23404 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Initialized, Role: FOLLOWER
18:21:19.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.404888 23404 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:19.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.405155 23404 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent starting tablet: real 0.004s	user 0.002s	sys 0.000s
18:21:19.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.405817 22636 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:19.407 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:19.408 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.130:37215
18:21:19.408 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.408681 23513 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.408735 23513 heartbeater.cc:461] Registering TS with master...
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.408797 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 23015
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.408820 23513 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.411331 23514 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.411383 23514 heartbeater.cc:461] Registering TS with master...
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.411465 23514 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.411473 22780 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.411789 22707 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.411988 22780 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.131:51849
18:21:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:19.412628 23513 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:19.416 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:19.416 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] handling channelInactive
18:21:19.416 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state READY due to: connection closed
18:21:19.425 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:19.426 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.426 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.426 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.447 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:19.448 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:19.448 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=3, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [10ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [31ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1888730822(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@654bd0a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@ae8a3a0 -> wakeup thread Time-limited test)) after lookup
18:21:19.449 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:19.450 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.450 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.450 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:19.467 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:19.468 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=5, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [10ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [31ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [32ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [34ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [34ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1888730822(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@654bd0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@ae8a3a0 -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011))) after lookup
18:21:19.468 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:19.469 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.470 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.470 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.527 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:19.527 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:19.528 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=7, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=111), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [10ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [31ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [32ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [34ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [34ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [52ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [54ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [54ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [110ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1888730822(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@654bd0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@ae8a3a0 -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941))) after lookup
18:21:19.529 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:19.530 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.530 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.530 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:19.588 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:19.588 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=9, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=172), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [10ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [31ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [32ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [34ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [34ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [52ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [54ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [54ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [110ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [112ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [114ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [114ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [171ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1888730822(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@654bd0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941) -> (continuation of Deferred@1266561154 after retry RPC@831629131), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@ae8a3a0 -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941) -> (continuation of Deferred@1266561154 after retry RPC@831629131))) after lookup
18:21:19.589 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:19.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:19.591 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:20.327 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:20.328 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:20.328 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=11, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=912), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [10ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [31ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [32ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [34ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [34ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [52ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [54ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [54ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [110ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [112ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [114ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [114ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [171ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [173ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [175ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [175ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [908ms] refreshing cache from master, [909ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [911ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1888730822(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@654bd0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941) -> (continuation of Deferred@1266561154 after retry RPC@831629131) -> (continuation of Deferred@1082215404 after retry RPC@1167745952), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@ae8a3a0 -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941) -> (continuation of Deferred@1266561154 after retry RPC@831629131) -> (continuation of Deferred@1082215404 after retry RPC@1167745952))) after lookup
18:21:20.329 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:20.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:20.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:20.331 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.774726 23525 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 86740cf8056941e1bca1ebcce92945d2)
18:21:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.774859 23525 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.775044 23525 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:20.775703 22895 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111) [suppressed 1 similar messages]
18:21:20.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:20.776285 22895 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.779994 23466 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" is_pre_election: true
18:21:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.780174 23466 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f6251917bba4996b6e74abedae69139 in term 1.
18:21:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.780467 22897 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 86740cf8056941e1bca1ebcce92945d2
18:21:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.780611 23525 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 3 FOLLOWER]: Leader pre-election won for term 4
18:21:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.780664 23525 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 86740cf8056941e1bca1ebcce92945d2)
18:21:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.780686 23525 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 3 FOLLOWER]: Advancing to term 4
18:21:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.781414 23525 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.781585 23525 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 4 election: Requested vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.781795 23466 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:20.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.781895 23466 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 1 FOLLOWER]: Advancing to term 4
18:21:20.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:20.782153 22895 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:20.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.782990 23466 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f6251917bba4996b6e74abedae69139 in term 4.
18:21:20.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.783231 22897 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 86740cf8056941e1bca1ebcce92945d2
18:21:20.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.783389 23525 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 4 FOLLOWER]: Leader election won for term 4
18:21:20.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.783546 23525 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 4 LEADER]: Becoming Leader. State: Replica: 7f6251917bba4996b6e74abedae69139, State: Running, Role: LEADER
18:21:20.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.783643 23525 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:20.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:20.784514 22780 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 reported cstate change: term changed from 3 to 4, leader changed from 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130) to 7f6251917bba4996b6e74abedae69139 (127.22.16.129). New cstate: current_term: 4 leader_uuid: "7f6251917bba4996b6e74abedae69139" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: UNKNOWN } } }
18:21:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:21.169664 23466 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 4 FOLLOWER]: Refusing update from remote peer 7f6251917bba4996b6e74abedae69139: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
18:21:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:21.170042 23525 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
18:21:21.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:21.173179 22895 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 -> Peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Couldn't send request to peer 86740cf8056941e1bca1ebcce92945d2. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:22.887 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:22.887 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:22.888 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=13, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=3471), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [0ms] delaying RPC due to: Illegal state: connection is terminated, [9ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [10ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [31ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [32ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [34ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [34ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [52ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [54ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [54ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [110ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [112ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [114ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [114ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [169ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [171ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [173ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [175ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [175ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [908ms] refreshing cache from master, [909ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [911ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [913ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [914ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [915ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3468ms] refreshing cache from master, [3469ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [3471ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1888730822(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@654bd0a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941) -> (continuation of Deferred@1266561154 after retry RPC@831629131) -> (continuation of Deferred@1082215404 after retry RPC@1167745952) -> (continuation of Deferred@155720464 after retry RPC@534376186), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@ae8a3a0 -> wakeup thread Time-limited test -> (continuation of Deferred@1637309842 after retry RPC@774703011) -> (continuation of Deferred@1608148858 after retry RPC@428634941) -> (continuation of Deferred@1266561154 after retry RPC@831629131) -> (continuation of Deferred@1082215404 after retry RPC@1167745952) -> (continuation of Deferred@155720464 after retry RPC@534376186))) after lookup
18:21:22.889 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:22.890 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] Successfully connected to peer
18:21:22.914 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xfe5ded18, L:/127.0.0.1:54224 - R:/127.22.16.129:41735] using TOKEN/null
18:21:22.928 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.130:37215
18:21:22.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:22.930433 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:37215
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.020457 23566 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:23.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.020673 23566 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.020707 23566 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.022341 23566 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.022399 23566 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.130
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.024230 23566 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:37215
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23566
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.024518 23566 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.024796 23566 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:23.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.027635 23572 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:23.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.027719 23571 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.027858 23566 server_base.cc:1061] running on GCE node
18:21:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:23.027680 23574 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.028230 23566 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.028400 23566 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.029531 23566 hybrid_clock.cc:648] HybridClock initialized: now 1783534883029499 us; error 47 us; skew 500 ppm
18:21:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.030516 23566 webserver.cc:534] Webserver started at http://127.22.16.130:33841/ using document root <none> and password file <none>
18:21:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.030668 23566 fs_manager.cc:362] Metadata directory not provided
18:21:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.030710 23566 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:23.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.031960 23566 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:21:23.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.032578 23580 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.032932 23566 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.032987 23566 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.033255 23566 fs_report.cc:389] FS layout report
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/data
18:21:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:23.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.053525 23566 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:23.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.053794 23566 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:23.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.053876 23566 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:23.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.054049 23566 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:23.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.054625 23587 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.055533 23566 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.055594 23566 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.055630 23566 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.056209 23566 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.056265 23566 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.056473 23587 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap starting.
18:21:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.063525 23566 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.130:37215
18:21:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.063570 23694 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.130:37215 every 8 connection(s)
18:21:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.063884 23566 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.066702 23587 log.cc:826] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Log is configured to *not* fsync() on all Append() calls
18:21:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.069005 23696 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.069085 23696 heartbeater.cc:461] Registering TS with master...
18:21:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.069268 23696 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.069718 23587 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.069856 22780 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:23.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.070097 23587 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap complete.
18:21:23.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.070218 22780 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.130:35061
18:21:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.071303 23587 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent bootstrapping tablet: real 0.015s	user 0.004s	sys 0.011s
18:21:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.072559 23587 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.072842 23587 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Initialized, Role: FOLLOWER
18:21:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.072850 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 23566
18:21:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.072963 23587 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:23.073 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:23.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.073203 23587 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.001s
18:21:23.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.073441 23696 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.076617 23695 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:23.077 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.076694 23695 heartbeater.cc:461] Registering TS with master...
18:21:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.076746 23695 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.077636 22635 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:23.078 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.129:41735
18:21:23.078 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:23.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.078933 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 22880
18:21:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.076633 23697 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.080157 23697 heartbeater.cc:461] Registering TS with master...
18:21:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.080226 23697 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:23.080904 22708 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:23.085 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:23.085 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] handling channelInactive
18:21:23.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state READY due to: connection closed
18:21:23.086 [DEBUG - Time-limited test] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:23.087 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.087 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.087 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.107 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:23.107 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:23.107 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@873090337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@312e1f48 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c6e8058 -> wakeup thread Time-limited test)) after lookup
18:21:23.108 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:23.109 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.109 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.109 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.127 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:23.128 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:23.128 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [23ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@873090337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@312e1f48 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c6e8058 -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837))) after lookup
18:21:23.129 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:23.130 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.130 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.130 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.168 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:23.168 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:23.169 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=83), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [23ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [44ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@873090337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@312e1f48 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c6e8058 -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002))) after lookup
18:21:23.169 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:23.170 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.170 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.171 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.208 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:23.208 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:23.208 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=123), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [23ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [44ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [84ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [86ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [86ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@873090337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@312e1f48 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c6e8058 -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520))) after lookup
18:21:23.209 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:23.210 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.210 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.210 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.667 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:23.667 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:23.668 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=583), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [23ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [44ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [84ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [86ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [86ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [124ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [125ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [125ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [579ms] refreshing cache from master, [579ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [582ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@873090337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@312e1f48 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520) -> (continuation of Deferred@1526531643 after retry RPC@170457059), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c6e8058 -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520) -> (continuation of Deferred@1526531643 after retry RPC@170457059))) after lookup
18:21:23.669 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:23.670 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.670 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:23.670 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:24.128 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:24.128 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:24.128 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1043), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [23ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [44ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [84ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [86ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [86ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [124ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [125ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [125ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [579ms] refreshing cache from master, [579ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [582ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [583ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [585ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [585ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [1040ms] refreshing cache from master, [1040ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1043ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@873090337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@312e1f48 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520) -> (continuation of Deferred@1526531643 after retry RPC@170457059) -> (continuation of Deferred@1852985841 after retry RPC@589833267), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c6e8058 -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520) -> (continuation of Deferred@1526531643 after retry RPC@170457059) -> (continuation of Deferred@1852985841 after retry RPC@589833267))) after lookup
18:21:24.129 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:24.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:24.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:24.130 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.428525 23707 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 7f6251917bba4996b6e74abedae69139)
18:21:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.428659 23707 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.428989 23707 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:24.429507 23398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:24.430123 23398 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:24.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.432744 23649 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2" is_pre_election: true
18:21:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.432922 23649 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 3.
18:21:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.433236 23398 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 86740cf8056941e1bca1ebcce92945d2; no voters: 7f6251917bba4996b6e74abedae69139
18:21:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.433393 23707 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 4 FOLLOWER]: Leader pre-election won for term 5
18:21:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.433465 23707 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 7f6251917bba4996b6e74abedae69139)
18:21:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.433511 23707 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 4 FOLLOWER]: Advancing to term 5
18:21:24.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.434286 23707 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:24.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.434446 23707 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 5 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:24.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.434796 23649 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:24.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.434907 23649 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 3 FOLLOWER]: Advancing to term 5
18:21:24.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:24.435139 23398 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.435993 23649 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 5.
18:21:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.436321 23398 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 86740cf8056941e1bca1ebcce92945d2; no voters: 7f6251917bba4996b6e74abedae69139
18:21:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.436467 23707 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 5 FOLLOWER]: Leader election won for term 5
18:21:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.436646 23707 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 5 LEADER]: Becoming Leader. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Running, Role: LEADER
18:21:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.436734 23707 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.437574 22780 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 reported cstate change: term changed from 4 to 5, leader changed from 7f6251917bba4996b6e74abedae69139 (127.22.16.129) to 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131). New cstate: current_term: 5 leader_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: HEALTHY } } }
18:21:24.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:24.857694 23398 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 -> Peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Couldn't send request to peer 7f6251917bba4996b6e74abedae69139. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:24.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.963569 23649 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 5 FOLLOWER]: Refusing update from remote peer 6f3eada3ec394bf9b1cc2e2a14c0e505: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
18:21:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:24.963917 23707 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
18:21:27.167 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:27.167 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:27.168 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=4082), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [23ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [44ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [83ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [84ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [86ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [86ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [124ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [125ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [125ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [579ms] refreshing cache from master, [579ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [582ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [583ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [585ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [585ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [1040ms] refreshing cache from master, [1040ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1043ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [1044ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1045ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [1045ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [4079ms] refreshing cache from master, [4080ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [4082ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@873090337(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@312e1f48 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520) -> (continuation of Deferred@1526531643 after retry RPC@170457059) -> (continuation of Deferred@1852985841 after retry RPC@589833267) -> (continuation of Deferred@173442599 after retry RPC@1757225698), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c6e8058 -> wakeup thread Time-limited test -> (continuation of Deferred@1477743575 after retry RPC@1623237837) -> (continuation of Deferred@1395710861 after retry RPC@917761002) -> (continuation of Deferred@138111278 after retry RPC@570310520) -> (continuation of Deferred@1526531643 after retry RPC@170457059) -> (continuation of Deferred@1852985841 after retry RPC@589833267) -> (continuation of Deferred@173442599 after retry RPC@1757225698))) after lookup
18:21:27.168 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:27.170 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] Successfully connected to peer
18:21:27.186 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xbecc4c48, L:/127.0.0.1:50870 - R:/127.22.16.131:42931] using TOKEN/null
18:21:27.196 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.129:41735
18:21:27.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.197621 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:41735
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44715
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.214428 23398 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 -> Peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Couldn't send request to peer 7f6251917bba4996b6e74abedae69139. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:21:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.280164 23748 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.280360 23748 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.280393 23748 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:27.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.282140 23748 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:27.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.282217 23748 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.129
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.283883 23748 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:41735
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44715
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23748
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.284143 23748 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.284425 23748 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.287436 23748 server_base.cc:1061] running on GCE node
18:21:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.287369 23754 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.287389 23756 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:27.287391 23753 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.287936 23748 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.288110 23748 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.289237 23748 hybrid_clock.cc:648] HybridClock initialized: now 1783534887289216 us; error 39 us; skew 500 ppm
18:21:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.290239 23748 webserver.cc:534] Webserver started at http://127.22.16.129:44715/ using document root <none> and password file <none>
18:21:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.290386 23748 fs_manager.cc:362] Metadata directory not provided
18:21:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.290452 23748 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:27.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.291572 23748 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.001s
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.292140 23762 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.292300 23748 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.001s
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.292340 23748 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f6251917bba4996b6e74abedae69139"
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.292587 23748 fs_report.cc:389] FS layout report
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/data
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.300177 23748 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.300406 23748 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.300501 23748 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.300684 23748 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.301095 23769 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.302121 23748 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.302165 23748 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.302189 23748 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.302672 23748 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.302701 23748 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.302810 23769 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap starting.
18:21:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.309742 23748 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.129:41735
18:21:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.309810 23876 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.129:41735 every 8 connection(s)
18:21:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.310149 23748 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.312264 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 23748
18:21:27.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.313288 23769 log.cc:826] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Log is configured to *not* fsync() on all Append() calls
18:21:27.317 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.318835 23769 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.319190 23769 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap complete.
18:21:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.320524 23769 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent bootstrapping tablet: real 0.018s	user 0.005s	sys 0.008s
18:21:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.320808 23878 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.320905 23878 heartbeater.cc:461] Registering TS with master...
18:21:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.321107 23878 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.321614 22781 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.322113 22781 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.129:50503
18:21:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.322206 23879 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.322268 23879 heartbeater.cc:461] Registering TS with master...
18:21:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.322358 23879 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.322654 22707 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.323081 23769 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.323336 23769 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f6251917bba4996b6e74abedae69139, State: Initialized, Role: FOLLOWER
18:21:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.323454 23769 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.323711 23769 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent starting tablet: real 0.003s	user 0.002s	sys 0.000s
18:21:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.323832 23878 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:27.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.325078 23877 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:27.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.325138 23877 heartbeater.cc:461] Registering TS with master...
18:21:27.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.325182 23877 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:27.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.325459 22636 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:27.331 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.131:42931
18:21:27.331 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:27.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:27.332120 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 23383
18:21:27.340 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:27.340 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] handling channelInactive
18:21:27.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state READY due to: connection closed
18:21:27.345 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:27.346 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.346 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.346 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.367 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:27.367 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:27.368 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [6ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [6ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1188093193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@60289496 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6368e5f0 -> wakeup thread Time-limited test)) after lookup
18:21:27.368 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:27.369 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.370 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.370 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.407 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:27.407 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:27.408 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [6ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [6ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [28ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [30ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [30ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1188093193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@60289496 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6368e5f0 -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752))) after lookup
18:21:27.408 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:27.409 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.409 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.410 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.446 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:27.447 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:27.447 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=107), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [6ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [6ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [28ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [30ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [30ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [68ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [69ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [70ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [106ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1188093193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@60289496 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6368e5f0 -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531))) after lookup
18:21:27.447 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:27.449 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.449 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.449 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.648 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:27.648 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:27.649 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=308), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [6ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [6ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [28ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [30ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [30ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [68ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [69ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [70ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [106ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [107ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [109ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [109ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [305ms] refreshing cache from master, [305ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [308ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1188093193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@60289496 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6368e5f0 -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830))) after lookup
18:21:27.650 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:27.651 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.651 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.651 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.747 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:27.747 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:27.747 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=407), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [6ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [6ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [28ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [30ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [30ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [68ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [69ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [70ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [106ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [107ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [109ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [109ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [305ms] refreshing cache from master, [305ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [308ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [309ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [311ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [311ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [404ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [406ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1188093193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@60289496 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830) -> (continuation of Deferred@1012089705 after retry RPC@2109210547), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6368e5f0 -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830) -> (continuation of Deferred@1012089705 after retry RPC@2109210547))) after lookup
18:21:27.748 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:27.749 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.749 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:27.749 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:28.007 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:28.007 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:28.008 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=667), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [6ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [6ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [28ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [30ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [30ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [68ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [69ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [70ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [106ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [107ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [109ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [109ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [305ms] refreshing cache from master, [305ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [308ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [309ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [311ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [311ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [404ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [406ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [408ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [409ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [409ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [664ms] refreshing cache from master, [665ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [667ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1188093193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@60289496 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830) -> (continuation of Deferred@1012089705 after retry RPC@2109210547) -> (continuation of Deferred@2037095688 after retry RPC@1004781553), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6368e5f0 -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830) -> (continuation of Deferred@1012089705 after retry RPC@2109210547) -> (continuation of Deferred@2037095688 after retry RPC@1004781553))) after lookup
18:21:28.008 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:28.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:28.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:28.010 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.696434 23890 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.696542 23890 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.696868 23890 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:28.697291 23583 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:28.697870 23583 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.700433 23831 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139" is_pre_election: true
18:21:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.700592 23831 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 4.
18:21:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.700866 23581 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.701032 23890 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 5 FOLLOWER]: Leader pre-election won for term 6
18:21:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.701092 23890 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.701108 23890 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 5 FOLLOWER]: Advancing to term 6
18:21:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.701678 23890 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.701823 23890 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 6 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:28.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.701974 23831 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139"
18:21:28.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.702072 23831 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 4 FOLLOWER]: Advancing to term 6
18:21:28.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:28.702389 23583 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:28.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.703138 23831 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 6.
18:21:28.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.703328 23581 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:28.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.703408 23890 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 6 FOLLOWER]: Leader election won for term 6
18:21:28.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.703537 23890 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 6 LEADER]: Becoming Leader. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Running, Role: LEADER
18:21:28.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.703603 23890 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:28.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:28.704376 22781 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 reported cstate change: term changed from 5 to 6, leader changed from 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131) to 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130). New cstate: current_term: 6 leader_uuid: "86740cf8056941e1bca1ebcce92945d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: UNKNOWN } } }
18:21:29.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.087968 23831 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 6 FOLLOWER]: Refusing update from remote peer 86740cf8056941e1bca1ebcce92945d2: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 17. (index mismatch)
18:21:29.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.088353 23890 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
18:21:29.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.091826 23583 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 -> Peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Couldn't send request to peer 6f3eada3ec394bf9b1cc2e2a14c0e505. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:29.387 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:29.387 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:29.388 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=15, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2047), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [0ms] delaying RPC due to: Illegal state: connection is terminated, [4ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [6ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [6ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [27ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [28ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [30ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [30ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [64ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [67ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [68ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [69ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [70ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [104ms] refreshing cache from master, [104ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [106ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [107ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [109ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [109ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [305ms] refreshing cache from master, [305ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [308ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [309ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [311ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [311ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [404ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [406ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [408ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [409ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [409ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [664ms] refreshing cache from master, [665ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [667ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [668ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [670ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [670ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2044ms] refreshing cache from master, [2044ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [2046ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1188093193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@60289496 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830) -> (continuation of Deferred@1012089705 after retry RPC@2109210547) -> (continuation of Deferred@2037095688 after retry RPC@1004781553) -> (continuation of Deferred@700411923 after retry RPC@252010210), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6368e5f0 -> wakeup thread Time-limited test -> (continuation of Deferred@725617781 after retry RPC@262351752) -> (continuation of Deferred@569748422 after retry RPC@932981531) -> (continuation of Deferred@751131710 after retry RPC@2060938830) -> (continuation of Deferred@1012089705 after retry RPC@2109210547) -> (continuation of Deferred@2037095688 after retry RPC@1004781553) -> (continuation of Deferred@700411923 after retry RPC@252010210))) after lookup
18:21:29.388 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:29.389 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] Successfully connected to peer
18:21:29.402 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x36e92941, L:/127.0.0.1:52644 - R:/127.22.16.130:37215] using TOKEN/null
18:21:29.411 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.131:42931
18:21:29.412 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.412652 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.491765 23918 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.491915 23918 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.491933 23918 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.493367 23918 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.493422 23918 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.131
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.494859 23918 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23918
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.495020 23918 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.495190 23918 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.498065 23923 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.498096 23924 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:29.498076 23926 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.498615 23918 server_base.cc:1061] running on GCE node
18:21:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.498770 23918 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:29.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.498979 23918 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:29.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.500133 23918 hybrid_clock.cc:648] HybridClock initialized: now 1783534889500119 us; error 39 us; skew 500 ppm
18:21:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.501101 23918 webserver.cc:534] Webserver started at http://127.22.16.131:41001/ using document root <none> and password file <none>
18:21:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.501245 23918 fs_manager.cc:362] Metadata directory not provided
18:21:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.501277 23918 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.502377 23918 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.502912 23932 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.503058 23918 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.503098 23918 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:12 on dist-test-slave-v72b"
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.503376 23918 fs_report.cc:389] FS layout report
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/data
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:29.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:29.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:29.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.523059 23918 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.523391 23918 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.523490 23918 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.523654 23918 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:29.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.524045 23939 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.524936 23918 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.524981 23918 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.525004 23918 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.525487 23918 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.525516 23918 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.525615 23939 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap starting.
18:21:29.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.532665 23918 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.131:42931
18:21:29.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.532975 23918 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:29.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.532985 24046 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.131:42931 every 8 connection(s)
18:21:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.535355 23939 log.cc:826] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Log is configured to *not* fsync() on all Append() calls
18:21:29.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.537638 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 23918
18:21:29.538 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:29.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.538708 23939 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:29.542 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:29.543 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.130:37215
18:21:29.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.539105 23939 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap complete.
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.539418 24047 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.539494 24047 heartbeater.cc:461] Registering TS with master...
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.539670 24047 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.540186 22635 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.540298 23939 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent bootstrapping tablet: real 0.015s	user 0.000s	sys 0.013s
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.541286 24049 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.541325 24049 heartbeater.cc:461] Registering TS with master...
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.541422 24049 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.541245 23939 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.541491 23939 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Initialized, Role: FOLLOWER
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.541580 23939 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.541785 23939 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent starting tablet: real 0.001s	user 0.000s	sys 0.002s
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.543547 22708 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.543665 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 23566
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.545521 24048 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.545583 24048 heartbeater.cc:461] Registering TS with master...
18:21:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.545666 24048 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:29.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.545938 22780 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:29.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.546365 22780 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.131:55003
18:21:29.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:29.546940 24048 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:29.551 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:29.551 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] handling channelInactive
18:21:29.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state READY due to: connection closed
18:21:29.552 [DEBUG - Time-limited test] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:29.553 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.553 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.553 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.567 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:29.567 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:29.567 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@986096495(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@390f3c14 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@220521c0 -> wakeup thread Time-limited test)) after lookup
18:21:29.568 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:29.569 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.569 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.569 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:29.587 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:29.588 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [17ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [18ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [18ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@986096495(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@390f3c14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@220521c0 -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531))) after lookup
18:21:29.588 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:29.590 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.590 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.590 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.631 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:29.632 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:29.632 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [17ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [18ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [18ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [37ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [39ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [39ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [80ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@986096495(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@390f3c14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@220521c0 -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467))) after lookup
18:21:29.633 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:29.634 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.634 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.634 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.727 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:29.727 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:29.727 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=176), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [17ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [18ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [18ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [37ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [39ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [39ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [80ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [83ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [83ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [173ms] refreshing cache from master, [173ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [175ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@986096495(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@390f3c14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@220521c0 -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702))) after lookup
18:21:29.728 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:29.729 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.729 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:29.729 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:30.227 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:30.227 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:30.227 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=675), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [17ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [18ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [18ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [37ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [39ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [39ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [80ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [83ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [83ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [173ms] refreshing cache from master, [173ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [175ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [177ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [178ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [178ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [673ms] refreshing cache from master, [673ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [675ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@986096495(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@390f3c14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702) -> (continuation of Deferred@2136636328 after retry RPC@1835904931), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@220521c0 -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702) -> (continuation of Deferred@2136636328 after retry RPC@1835904931))) after lookup
18:21:30.228 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:30.229 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:30.229 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:30.229 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:30.907 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:30.907 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:30.908 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1356), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [17ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [18ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [18ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [37ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [39ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [39ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [80ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [83ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [83ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [173ms] refreshing cache from master, [173ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [175ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [177ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [178ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [178ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [673ms] refreshing cache from master, [673ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [675ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [677ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [678ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [678ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [1354ms] refreshing cache from master, [1354ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1356ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@986096495(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@390f3c14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702) -> (continuation of Deferred@2136636328 after retry RPC@1835904931) -> (continuation of Deferred@688553680 after retry RPC@689626341), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@220521c0 -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702) -> (continuation of Deferred@2136636328 after retry RPC@1835904931) -> (continuation of Deferred@688553680 after retry RPC@689626341))) after lookup
18:21:30.908 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:30.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:30.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:30.909 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.912122 24061 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 86740cf8056941e1bca1ebcce92945d2)
18:21:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.912235 24061 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.912586 24061 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:30.913064 23763 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:30.913714 23763 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:30.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.916357 24001 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" is_pre_election: true
18:21:30.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.916540 24001 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f6251917bba4996b6e74abedae69139 in term 5.
18:21:30.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.916795 23765 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 86740cf8056941e1bca1ebcce92945d2
18:21:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.916925 24061 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 6 FOLLOWER]: Leader pre-election won for term 7
18:21:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.916996 24061 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 86740cf8056941e1bca1ebcce92945d2)
18:21:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.917026 24061 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 6 FOLLOWER]: Advancing to term 7
18:21:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.917784 24061 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.917949 24061 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 7 election: Requested vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.918133 24001 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.918251 24001 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 5 FOLLOWER]: Advancing to term 7
18:21:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:30.918531 23763 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.919363 24001 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f6251917bba4996b6e74abedae69139 in term 7.
18:21:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.919549 23765 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 86740cf8056941e1bca1ebcce92945d2
18:21:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.919663 24061 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 7 FOLLOWER]: Leader election won for term 7
18:21:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.919802 24061 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 7 LEADER]: Becoming Leader. State: Replica: 7f6251917bba4996b6e74abedae69139, State: Running, Role: LEADER
18:21:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.919898 24061 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:30.920643 22780 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 reported cstate change: term changed from 6 to 7, leader changed from 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130) to 7f6251917bba4996b6e74abedae69139 (127.22.16.129). New cstate: current_term: 7 leader_uuid: "7f6251917bba4996b6e74abedae69139" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: UNKNOWN } } }
18:21:31.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:31.324427 24001 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 7 FOLLOWER]: Refusing update from remote peer 7f6251917bba4996b6e74abedae69139: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
18:21:31.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:31.324775 24061 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
18:21:31.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:31.327680 23763 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 -> Peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Couldn't send request to peer 86740cf8056941e1bca1ebcce92945d2. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:32.047 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:32.048 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:32.048 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2496), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [16ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [17ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [18ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [18ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [33ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [35ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [37ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [39ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [39ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [80ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [81ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [83ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [83ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [173ms] refreshing cache from master, [173ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [175ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [177ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [178ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [178ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [673ms] refreshing cache from master, [673ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [675ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [677ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [678ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [678ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [1354ms] refreshing cache from master, [1354ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1356ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [1357ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [1358ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [1358ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [2494ms] refreshing cache from master, [2494ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [2496ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@986096495(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@390f3c14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702) -> (continuation of Deferred@2136636328 after retry RPC@1835904931) -> (continuation of Deferred@688553680 after retry RPC@689626341) -> (continuation of Deferred@683192837 after retry RPC@1471856131), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@220521c0 -> wakeup thread Time-limited test -> (continuation of Deferred@1034072374 after retry RPC@916626531) -> (continuation of Deferred@19617761 after retry RPC@591954467) -> (continuation of Deferred@1575732762 after retry RPC@978652702) -> (continuation of Deferred@2136636328 after retry RPC@1835904931) -> (continuation of Deferred@688553680 after retry RPC@689626341) -> (continuation of Deferred@683192837 after retry RPC@1471856131))) after lookup
18:21:32.049 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:32.050 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] Successfully connected to peer
18:21:32.067 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xef484c2a, L:/127.0.0.1:47776 - R:/127.22.16.129:41735] using TOKEN/null
18:21:32.085 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.130:37215
18:21:32.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:32.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.087620 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:32.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:32.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:37215
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.195264 24095 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.195444 24095 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.195466 24095 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.197014 24095 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.197075 24095 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.130
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.198591 24095 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:37215
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24095
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.198782 24095 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.198978 24095 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.201602 24100 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.201603 24101 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.201679 24095 server_base.cc:1061] running on GCE node
18:21:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:32.201588 24103 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.202009 24095 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.202210 24095 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.203366 24095 hybrid_clock.cc:648] HybridClock initialized: now 1783534892203348 us; error 35 us; skew 500 ppm
18:21:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.204551 24095 webserver.cc:534] Webserver started at http://127.22.16.130:33841/ using document root <none> and password file <none>
18:21:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.204780 24095 fs_manager.cc:362] Metadata directory not provided
18:21:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.204854 24095 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.206064 24095 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:21:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.206727 24109 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.206935 24095 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.000s
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.207022 24095 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.207360 24095 fs_report.cc:389] FS layout report
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/data
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.221380 24095 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:32.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.221731 24095 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.221894 24095 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.222151 24095 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.222613 24116 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.223459 24095 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.223517 24095 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.223551 24095 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.224170 24095 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.224224 24095 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.224279 24116 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap starting.
18:21:32.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.230944 24095 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.130:37215
18:21:32.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.231000 24223 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.130:37215 every 8 connection(s)
18:21:32.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.231375 24095 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:32.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.232789 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 24095
18:21:32.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.236992 24116 log.cc:826] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Log is configured to *not* fsync() on all Append() calls
18:21:32.237 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:32.238 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.129:41735
18:21:32.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.238639 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 23748
18:21:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.240474 24224 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.240568 24224 heartbeater.cc:461] Registering TS with master...
18:21:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.240763 24224 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.241600 24116 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.241847 24116 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap complete.
18:21:32.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.242763 24116 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent bootstrapping tablet: real 0.019s	user 0.003s	sys 0.008s
18:21:32.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.243620 22635 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.243844 24116 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.244182 24116 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Initialized, Role: FOLLOWER
18:21:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.244380 24225 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.244436 24225 heartbeater.cc:461] Registering TS with master...
18:21:32.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.244462 24116 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:32.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.244545 24225 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:32.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.244804 22781 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:32.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.245253 22781 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.130:37225
18:21:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.245539 24116 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent starting tablet: real 0.003s	user 0.000s	sys 0.005s
18:21:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.245886 24225 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:32.246 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:32.246 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] handling channelInactive
18:21:32.247 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state READY due to: connection closed
18:21:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.247699 24226 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.247759 24226 heartbeater.cc:461] Registering TS with master...
18:21:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.247812 24226 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:32.248121 22708 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:32.248 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: connection closed
18:21:32.268 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:32.268 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:32.268 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@841289798(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@11d395b8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ad75aa7 -> wakeup thread Time-limited test)) after lookup
18:21:32.268 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:32.270 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.270 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.270 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.288 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:32.288 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:32.289 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [22ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@841289798(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@11d395b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ad75aa7 -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511))) after lookup
18:21:32.290 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:32.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.291 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.328 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:32.328 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:32.328 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [22ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [43ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [81ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@841289798(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@11d395b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ad75aa7 -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010))) after lookup
18:21:32.329 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:32.330 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.330 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.330 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.447 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:32.447 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:32.447 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=200), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [22ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [43ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [81ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [83ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [84ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [84ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [200ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@841289798(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@11d395b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010) -> (continuation of Deferred@1685836447 after retry RPC@1496096545), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ad75aa7 -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010) -> (continuation of Deferred@1685836447 after retry RPC@1496096545))) after lookup
18:21:32.448 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:32.449 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.449 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.449 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.927 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:32.928 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:32.928 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=681), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [22ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [43ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [81ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [83ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [84ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [84ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [200ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [202ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [203ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [203ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [679ms] refreshing cache from master, [679ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [681ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@841289798(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@11d395b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010) -> (continuation of Deferred@1685836447 after retry RPC@1496096545) -> (continuation of Deferred@1836155846 after retry RPC@1352260830), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ad75aa7 -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010) -> (continuation of Deferred@1685836447 after retry RPC@1496096545) -> (continuation of Deferred@1836155846 after retry RPC@1352260830))) after lookup
18:21:32.929 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:32.930 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.930 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:32.930 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.586184 24237 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 7f6251917bba4996b6e74abedae69139)
18:21:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.586294 24237 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.586602 24237 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:33.587129 23933 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:33.587668 23933 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.590373 24178 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2" is_pre_election: true
18:21:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.590497 24178 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 6.
18:21:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.590701 23933 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 86740cf8056941e1bca1ebcce92945d2; no voters: 7f6251917bba4996b6e74abedae69139
18:21:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.590857 24237 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 7 FOLLOWER]: Leader pre-election won for term 8
18:21:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.590934 24237 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 7f6251917bba4996b6e74abedae69139)
18:21:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.590966 24237 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 7 FOLLOWER]: Advancing to term 8
18:21:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.591746 24237 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.591905 24237 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 8 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.592157 24178 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.592226 24178 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 6 FOLLOWER]: Advancing to term 8
18:21:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.593144 24178 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 8.
18:21:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.593307 23933 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 86740cf8056941e1bca1ebcce92945d2; no voters: 
18:21:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.593392 24237 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 8 FOLLOWER]: Leader election won for term 8
18:21:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.593556 24237 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 8 LEADER]: Becoming Leader. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Running, Role: LEADER
18:21:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:33.593573 23933 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.593659 24237 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:33.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:33.594345 22781 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 reported cstate change: term changed from 7 to 8, leader changed from 7f6251917bba4996b6e74abedae69139 (127.22.16.129) to 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131). New cstate: current_term: 8 leader_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: HEALTHY } } }
18:21:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:33.971935 23933 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 -> Peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Couldn't send request to peer 7f6251917bba4996b6e74abedae69139. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:34.167 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:34.167 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:34.168 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1921), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [2ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [21ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [22ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [24ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [24ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [42ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [43ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [45ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [45ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [81ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [83ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [84ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [84ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [198ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [200ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [202ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [203ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [203ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [679ms] refreshing cache from master, [679ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [681ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [682ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [684ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [684ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [1918ms] refreshing cache from master, [1919ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1921ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@841289798(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@11d395b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010) -> (continuation of Deferred@1685836447 after retry RPC@1496096545) -> (continuation of Deferred@1836155846 after retry RPC@1352260830) -> (continuation of Deferred@1507769246 after retry RPC@1203762007), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ad75aa7 -> wakeup thread Time-limited test -> (continuation of Deferred@1870948890 after retry RPC@793118511) -> (continuation of Deferred@358021488 after retry RPC@179760010) -> (continuation of Deferred@1685836447 after retry RPC@1496096545) -> (continuation of Deferred@1836155846 after retry RPC@1352260830) -> (continuation of Deferred@1507769246 after retry RPC@1203762007))) after lookup
18:21:34.168 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:34.169 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] Successfully connected to peer
18:21:34.195 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xe0ccc5fa, L:/127.0.0.1:50910 - R:/127.22.16.131:42931] using TOKEN/null
18:21:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.197185 24178 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 8 FOLLOWER]: Refusing update from remote peer 6f3eada3ec394bf9b1cc2e2a14c0e505: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 26. (index mismatch)
18:21:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.197744 24257 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
18:21:34.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.200369 24256 mvcc.cc:204] Tried to move back new op lower bound from 7305358926629416960 to 7305358924159946752. Current Snapshot: MvccSnapshot[applied={T|T < 7305358926629416960}]
18:21:34.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.201256 24259 mvcc.cc:204] Tried to move back new op lower bound from 7305358926629416960 to 7305358924159946752. Current Snapshot: MvccSnapshot[applied={T|T < 7305358926629416960 or (T in {7305358926629416960})}]
18:21:34.208 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.129:41735
18:21:34.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.208791 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:41735
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44715
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:34.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.284983 24266 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:34.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.285142 24266 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:34.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.285161 24266 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:34.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.286660 24266 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:34.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.286710 24266 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.129
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.288226 24266 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:41735
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44715
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24266
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.288424 24266 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:34.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.288618 24266 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.291078 24272 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.291106 24266 server_base.cc:1061] running on GCE node
18:21:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.291114 24271 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:34.291090 24274 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.291492 24266 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.291687 24266 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.292809 24266 hybrid_clock.cc:648] HybridClock initialized: now 1783534894292796 us; error 31 us; skew 500 ppm
18:21:34.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.293792 24266 webserver.cc:534] Webserver started at http://127.22.16.129:44715/ using document root <none> and password file <none>
18:21:34.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.293936 24266 fs_manager.cc:362] Metadata directory not provided
18:21:34.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.293970 24266 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:34.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.295449 24266 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.296026 24280 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.296175 24266 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.296239 24266 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f6251917bba4996b6e74abedae69139"
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.296481 24266 fs_report.cc:389] FS layout report
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/data
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:34.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:34.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.315692 24266 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.315934 24266 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.316017 24266 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.316188 24266 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.316711 24287 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:34.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.317577 24266 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:34.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.317754 24266 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.317797 24266 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.318420 24266 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.318475 24266 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.318518 24287 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap starting.
18:21:34.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.324466 24266 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.129:41735
18:21:34.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.324517 24394 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.129:41735 every 8 connection(s)
18:21:34.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.324798 24266 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:34.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.328969 24287 log.cc:826] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Log is configured to *not* fsync() on all Append() calls
18:21:34.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.332304 24287 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:34.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.332628 24287 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap complete.
18:21:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.333382 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 24266
18:21:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.333657 24287 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent bootstrapping tablet: real 0.015s	user 0.003s	sys 0.009s
18:21:34.334 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:34.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.334848 24287 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:34.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.335114 24287 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f6251917bba4996b6e74abedae69139, State: Initialized, Role: FOLLOWER
18:21:34.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.335278 24287 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:34.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.335552 24287 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.004s
18:21:34.338 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:34.338 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.131:42931
18:21:34.339 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.339180 24395 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.339294 24395 heartbeater.cc:461] Registering TS with master...
18:21:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.339445 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 23918
18:21:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.339488 24395 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.340904 24396 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.340987 24396 heartbeater.cc:461] Registering TS with master...
18:21:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.341095 24396 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.341147 22635 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.341588 22781 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.342094 22781 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.129:57511
18:21:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.342851 24396 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.342981 24397 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.343159 24397 heartbeater.cc:461] Registering TS with master...
18:21:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.343218 24397 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:34.343533 22708 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:34.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:34.345 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] handling channelInactive
18:21:34.346 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state READY due to: connection closed
18:21:34.365 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:34.366 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.366 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.387 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:34.387 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:34.387 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=3, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [41ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@923938562(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@f0f57fc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@290c35f8 -> wakeup thread Time-limited test)) after lookup
18:21:34.388 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:34.388 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.388 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.389 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.406 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:34.406 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:34.406 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=5, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [41ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [42ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [44ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@923938562(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@f0f57fc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@290c35f8 -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738))) after lookup
18:21:34.407 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:34.408 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.408 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.408 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:34.467 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:34.468 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=7, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=122), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [41ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [42ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [44ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [62ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [63ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [63ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@923938562(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@f0f57fc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@290c35f8 -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026))) after lookup
18:21:34.468 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:34.469 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.469 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.469 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:34.587 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:34.588 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=9, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=242), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [41ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [42ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [44ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [62ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [63ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [63ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [123ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [124ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [124ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [239ms] refreshing cache from master, [240ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [242ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@923938562(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@f0f57fc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026) -> (continuation of Deferred@688583187 after retry RPC@129714320), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@290c35f8 -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026) -> (continuation of Deferred@688583187 after retry RPC@129714320))) after lookup
18:21:34.588 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:34.589 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.590 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:34.590 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:35.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:35.587 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:35.587 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=11, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1242), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [41ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [42ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [44ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [62ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [63ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [63ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [123ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [124ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [124ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [239ms] refreshing cache from master, [240ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [242ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [243ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [245ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [245ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [1240ms] refreshing cache from master, [1240ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1242ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@923938562(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@f0f57fc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026) -> (continuation of Deferred@688583187 after retry RPC@129714320) -> (continuation of Deferred@1863418229 after retry RPC@850639823), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@290c35f8 -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026) -> (continuation of Deferred@688583187 after retry RPC@129714320) -> (continuation of Deferred@1863418229 after retry RPC@850639823))) after lookup
18:21:35.588 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:35.589 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:35.589 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:35.589 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.708107 24408 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.708194 24408 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.708814 24408 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:35.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:35.709147 24112 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:35.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:35.709707 24112 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.713558 24349 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139" is_pre_election: true
18:21:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.713730 24349 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 7.
18:21:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.713984 24110 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.714146 24408 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 8 FOLLOWER]: Leader pre-election won for term 9
18:21:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.714233 24408 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.714265 24408 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 8 FOLLOWER]: Advancing to term 9
18:21:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.714931 24408 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.715299 24408 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 9 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.715309 24349 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139"
18:21:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.715389 24349 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 7 FOLLOWER]: Advancing to term 9
18:21:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:35.715974 24112 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.716616 24349 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 9.
18:21:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.716804 24110 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.716902 24408 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 9 FOLLOWER]: Leader election won for term 9
18:21:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.717067 24408 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 9 LEADER]: Becoming Leader. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Running, Role: LEADER
18:21:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.717155 24408 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:35.717931 22781 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 reported cstate change: term changed from 8 to 9, leader changed from 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131) to 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130). New cstate: current_term: 9 leader_uuid: "86740cf8056941e1bca1ebcce92945d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: UNKNOWN } } }
18:21:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.099020 24112 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 -> Peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Couldn't send request to peer 6f3eada3ec394bf9b1cc2e2a14c0e505. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:36.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.238962 24349 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 9 FOLLOWER]: Refusing update from remote peer 86740cf8056941e1bca1ebcce92945d2: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
18:21:36.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.239288 24408 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
18:21:36.646 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:36.647 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:36.647 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=13, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2301), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [1ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [41ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [42ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [44ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [44ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [59ms] refreshing cache from master, [59ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [61ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [62ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [63ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [63ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [122ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [123ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [124ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [124ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [239ms] refreshing cache from master, [240ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [242ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [243ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [245ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [245ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [1240ms] refreshing cache from master, [1240ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1242ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [1243ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [1244ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [1244ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2299ms] refreshing cache from master, [2299ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [2301ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@923938562(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@f0f57fc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026) -> (continuation of Deferred@688583187 after retry RPC@129714320) -> (continuation of Deferred@1863418229 after retry RPC@850639823) -> (continuation of Deferred@1609424834 after retry RPC@1882159667), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@290c35f8 -> wakeup thread Time-limited test -> (continuation of Deferred@834484251 after retry RPC@251078738) -> (continuation of Deferred@1142535675 after retry RPC@1939579026) -> (continuation of Deferred@688583187 after retry RPC@129714320) -> (continuation of Deferred@1863418229 after retry RPC@850639823) -> (continuation of Deferred@1609424834 after retry RPC@1882159667))) after lookup
18:21:36.648 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:36.649 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] Successfully connected to peer
18:21:36.677 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xf746edde, L:/127.0.0.1:53380 - R:/127.22.16.130:37215] using TOKEN/null
18:21:36.694 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.131:42931
18:21:36.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.695133 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:36.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.775494 24437 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:36.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.775655 24437 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:36.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.775673 24437 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:36.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.777081 24437 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:36.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.777127 24437 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.131
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.779212 24437 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24437
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.779455 24437 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.779682 24437 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:36.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.782294 24443 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:36.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.782326 24442 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:36.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:36.782526 24445 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:36.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.783208 24437 server_base.cc:1061] running on GCE node
18:21:36.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.783420 24437 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:36.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.783617 24437 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:36.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.784765 24437 hybrid_clock.cc:648] HybridClock initialized: now 1783534896784754 us; error 28 us; skew 500 ppm
18:21:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.786046 24437 webserver.cc:534] Webserver started at http://127.22.16.131:41001/ using document root <none> and password file <none>
18:21:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.786283 24437 fs_manager.cc:362] Metadata directory not provided
18:21:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.786365 24437 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:36.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.787571 24437 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:21:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.788210 24451 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.788375 24437 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
18:21:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.788446 24437 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:12 on dist-test-slave-v72b"
18:21:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.788724 24437 fs_report.cc:389] FS layout report
18:21:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/data
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.800762 24437 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.801091 24437 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.801259 24437 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.801486 24437 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.802022 24458 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.802871 24437 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.802918 24437 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.802949 24437 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.803661 24437 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.803705 24437 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.803732 24458 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap starting.
18:21:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.811743 24437 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.131:42931
18:21:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.812278 24437 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.813129 24565 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.131:42931 every 8 connection(s)
18:21:36.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.815138 24458 log.cc:826] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Log is configured to *not* fsync() on all Append() calls
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.818945 24567 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819042 24566 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819056 24567 heartbeater.cc:461] Registering TS with master...
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819108 24566 heartbeater.cc:461] Registering TS with master...
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819314 24566 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819319 24567 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819628 24458 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819847 22780 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.819885 22635 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.820274 22780 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.131:49109
18:21:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.820457 24458 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap complete.
18:21:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.820525 24568 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.820567 24568 heartbeater.cc:461] Registering TS with master...
18:21:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.820678 24568 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:36.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.821157 22708 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:36.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.821403 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 24437
18:21:36.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.821635 24458 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent bootstrapping tablet: real 0.018s	user 0.005s	sys 0.010s
18:21:36.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.822548 24458 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:36.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.822754 24458 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Initialized, Role: FOLLOWER
18:21:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.823401 24458 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:36.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.823661 24458 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent starting tablet: real 0.002s	user 0.003s	sys 0.000s
18:21:36.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.823808 24567 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:36.825 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:36.826 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.130:37215
18:21:36.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:36.827008 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 24095
18:21:36.833 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:36.834 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] handling channelInactive
18:21:36.834 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state READY due to: connection closed
18:21:36.835 [DEBUG - Time-limited test] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:36.836 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.836 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.837 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.846 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:36.846 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:36.847 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [12ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1269650975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@40d843bb -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63d0d6e9 -> wakeup thread Time-limited test)) after lookup
18:21:36.847 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:36.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.849 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.867 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:36.867 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:36.867 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [12ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [13ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [15ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [15ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [32ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1269650975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@40d843bb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63d0d6e9 -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013))) after lookup
18:21:36.868 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:36.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.869 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.907 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:36.907 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:36.907 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [12ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [13ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [15ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [15ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [32ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [33ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [35ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [35ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [72ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1269650975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@40d843bb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63d0d6e9 -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744))) after lookup
18:21:36.908 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:36.908 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.909 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:36.909 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:37.007 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:37.007 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:37.008 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=173), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [12ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [13ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [15ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [15ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [32ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [33ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [35ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [35ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [72ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [73ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [75ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [75ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [173ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1269650975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@40d843bb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63d0d6e9 -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607))) after lookup
18:21:37.008 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:37.009 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:37.009 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:37.009 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:37.266 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:37.267 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:37.267 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=433), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [12ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [13ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [15ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [15ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [32ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [33ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [35ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [35ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [72ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [73ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [75ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [75ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [173ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [174ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [175ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [175ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [430ms] refreshing cache from master, [430ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [432ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1269650975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@40d843bb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607) -> (continuation of Deferred@2044601954 after retry RPC@2024897637), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63d0d6e9 -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607) -> (continuation of Deferred@2044601954 after retry RPC@2024897637))) after lookup
18:21:37.268 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:37.269 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:37.269 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:37.269 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:38.147 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:38.147 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:38.147 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1313), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [12ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [13ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [15ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [15ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [32ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [33ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [35ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [35ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [72ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [73ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [75ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [75ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [173ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [174ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [175ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [175ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [430ms] refreshing cache from master, [430ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [432ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [434ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [435ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [435ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [1310ms] refreshing cache from master, [1311ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1312ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1269650975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@40d843bb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607) -> (continuation of Deferred@2044601954 after retry RPC@2024897637) -> (continuation of Deferred@452615851 after retry RPC@1767594949), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63d0d6e9 -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607) -> (continuation of Deferred@2044601954 after retry RPC@2024897637) -> (continuation of Deferred@452615851 after retry RPC@1767594949))) after lookup
18:21:38.148 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:38.149 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:38.149 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state CONNECTING due to: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:38.149 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215
18:21:38.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.192761 24579 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 86740cf8056941e1bca1ebcce92945d2)
18:21:38.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.192879 24579 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:38.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.193174 24579 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:38.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.193651 24281 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.194367 24281 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:38.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.196578 24515 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" is_pre_election: true
18:21:38.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.196738 24515 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f6251917bba4996b6e74abedae69139 in term 8.
18:21:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.197012 24283 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 86740cf8056941e1bca1ebcce92945d2
18:21:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.197196 24579 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 9 FOLLOWER]: Leader pre-election won for term 10
18:21:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.197264 24579 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 86740cf8056941e1bca1ebcce92945d2)
18:21:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.197289 24579 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 9 FOLLOWER]: Advancing to term 10
18:21:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.197964 24579 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.198114 24579 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 10 election: Requested vote from peers 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.198355 24515 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "7f6251917bba4996b6e74abedae69139" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.198464 24515 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 8 FOLLOWER]: Advancing to term 10
18:21:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.198650 24281 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111)
18:21:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.199608 24515 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f6251917bba4996b6e74abedae69139 in term 10.
18:21:38.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.199824 24283 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 7f6251917bba4996b6e74abedae69139; no voters: 86740cf8056941e1bca1ebcce92945d2
18:21:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.199942 24579 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 10 FOLLOWER]: Leader election won for term 10
18:21:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.200088 24579 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 10 LEADER]: Becoming Leader. State: Replica: 7f6251917bba4996b6e74abedae69139, State: Running, Role: LEADER
18:21:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.200171 24579 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.200897 22780 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 reported cstate change: term changed from 9 to 10, leader changed from 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130) to 7f6251917bba4996b6e74abedae69139 (127.22.16.129). New cstate: current_term: 10 leader_uuid: "7f6251917bba4996b6e74abedae69139" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: UNKNOWN } } }
18:21:38.247 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:38.247 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:38.248 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1413), Traces: [0ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [2ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [3ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [12ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [13ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [15ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [15ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [32ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [33ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [35ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [35ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [72ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [73ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [75ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [75ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [173ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [174ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [175ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [175ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [430ms] refreshing cache from master, [430ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [432ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [434ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [435ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [435ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [1310ms] refreshing cache from master, [1311ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1312ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [1314ms] sending RPC to server 86740cf8056941e1bca1ebcce92945d2, [1315ms] received response from server 86740cf8056941e1bca1ebcce92945d2: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [1315ms] delaying RPC due to: Network error: Failed to connect to peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215): Connection refused: /127.22.16.130:37215, [1411ms] refreshing cache from master, [1411ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1413ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1269650975(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@40d843bb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607) -> (continuation of Deferred@2044601954 after retry RPC@2024897637) -> (continuation of Deferred@452615851 after retry RPC@1767594949) -> (continuation of Deferred@904250921 after retry RPC@106116643), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63d0d6e9 -> wakeup thread Time-limited test -> (continuation of Deferred@1183321083 after retry RPC@587922013) -> (continuation of Deferred@1150867827 after retry RPC@1888763744) -> (continuation of Deferred@1623744321 after retry RPC@829817607) -> (continuation of Deferred@2044601954 after retry RPC@2024897637) -> (continuation of Deferred@452615851 after retry RPC@1767594949) -> (continuation of Deferred@904250921 after retry RPC@106116643))) after lookup
18:21:38.248 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:38.249 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] Successfully connected to peer
18:21:38.262 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x9f0e2016, L:/127.0.0.1:48812 - R:/127.22.16.129:41735] using TOKEN/null
18:21:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.264106 24515 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 10 FOLLOWER]: Refusing update from remote peer 7f6251917bba4996b6e74abedae69139: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 34. (index mismatch)
18:21:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.264328 24281 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 -> Peer 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215): Couldn't send request to peer 86740cf8056941e1bca1ebcce92945d2. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.130:37215: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.264424 24584 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
18:21:38.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.265803 24599 mvcc.cc:204] Tried to move back new op lower bound from 7305358936796528640 to 7305358932858019840. Current Snapshot: MvccSnapshot[applied={T|T < 7305358936796528640}]
18:21:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.267877 24597 mvcc.cc:204] Tried to move back new op lower bound from 7305358943287054336 to 7305358943028314112. Current Snapshot: MvccSnapshot[applied={T|T < 7305358943287054336 or (T in {7305358943287054336})}]
18:21:38.274 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.130:37215
18:21:38.275 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.275435 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:37215
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.355491 24606 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.355674 24606 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.355705 24606 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.357354 24606 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.357434 24606 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.130
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.359021 24606 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.130:37215
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.130
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24606
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.130
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/logs
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.359294 24606 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.359578 24606 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:38.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.362242 24612 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:38.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.362287 24614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:38.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:38.362371 24611 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:38.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.362792 24606 server_base.cc:1061] running on GCE node
18:21:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.362937 24606 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.363082 24606 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.364229 24606 hybrid_clock.cc:648] HybridClock initialized: now 1783534898364205 us; error 35 us; skew 500 ppm
18:21:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.366030 24606 webserver.cc:534] Webserver started at http://127.22.16.130:33841/ using document root <none> and password file <none>
18:21:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.366250 24606 fs_manager.cc:362] Metadata directory not provided
18:21:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.366334 24606 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:38.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.367831 24606 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.002s	sys 0.000s
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.368808 24620 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.369081 24606 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.369153 24606 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.369451 24606 fs_report.cc:389] FS layout report
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/wal
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/data
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.380792 24606 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.381116 24606 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.381240 24606 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.381502 24606 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.382073 24627 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.383209 24606 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.383464 24606 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:21:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.383630 24606 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.384588 24606 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.384704 24627 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap starting.
18:21:38.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.384807 24606 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
18:21:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.393319 24606 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.130:37215
18:21:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.393394 24734 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.130:37215 every 8 connection(s)
18:21:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.393752 24606 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-1/data/info.pb
18:21:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.395165 24627 log.cc:826] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Log is configured to *not* fsync() on all Append() calls
18:21:38.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.399945 24627 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:38.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.400611 24627 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Bootstrap complete.
18:21:38.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.401379 24735 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.401489 24735 heartbeater.cc:461] Registering TS with master...
18:21:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.401841 24735 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.402271 24627 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent bootstrapping tablet: real 0.018s	user 0.007s	sys 0.009s
18:21:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.401388 24736 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.402631 24736 heartbeater.cc:461] Registering TS with master...
18:21:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.402626 22635 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.402892 24736 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:38.403 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.403205 22780 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.403465 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 24606
18:21:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.403725 22780 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.130:52555
18:21:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.403535 24627 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.404790 24627 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Initialized, Role: FOLLOWER
18:21:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.404922 24627 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.405167 24627 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2: Time spent starting tablet: real 0.003s	user 0.002s	sys 0.000s
18:21:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.406356 24736 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:38.407 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:38.408 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.129:41735
18:21:38.408 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.408967 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 24266
18:21:38.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:38.417 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] handling channelInactive
18:21:38.417 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state READY due to: connection closed
18:21:38.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.419740 24737 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.419831 24737 heartbeater.cc:461] Registering TS with master...
18:21:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.419888 24737 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:38.420421 22708 ts_manager.cc:194] Re-registered known tserver with Master: 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:38.425 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:38.426 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.426 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.426 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.447 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:38.447 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:38.447 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=3, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1ms] delaying RPC due to: Illegal state: connection is terminated, [7ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [9ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [9ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [29ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@367096678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@a9cbdd7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d17775b -> wakeup thread Time-limited test)) after lookup
18:21:38.448 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:38.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.449 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:38.467 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:38.468 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=5, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1ms] delaying RPC due to: Illegal state: connection is terminated, [7ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [9ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [9ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [29ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [31ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [32ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@367096678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@a9cbdd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d17775b -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725))) after lookup
18:21:38.468 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:38.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.469 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.547 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:38.547 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:38.548 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=7, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=130), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1ms] delaying RPC due to: Illegal state: connection is terminated, [7ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [9ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [9ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [29ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [31ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [32ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [51ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [52ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [52ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [130ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@367096678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@a9cbdd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d17775b -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592))) after lookup
18:21:38.548 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:38.549 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.549 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.549 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.647 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:38.647 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:38.648 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=9, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=230), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1ms] delaying RPC due to: Illegal state: connection is terminated, [7ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [9ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [9ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [29ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [31ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [32ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [51ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [52ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [52ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [130ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [131ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [132ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [132ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [230ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@367096678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@a9cbdd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592) -> (continuation of Deferred@998565993 after retry RPC@1761113615), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d17775b -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592) -> (continuation of Deferred@998565993 after retry RPC@1761113615))) after lookup
18:21:38.648 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:38.649 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.649 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:38.649 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:39.647 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:39.647 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)[L],86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:39.648 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=11, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1230), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1ms] delaying RPC due to: Illegal state: connection is terminated, [7ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [9ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [9ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [29ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [31ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [32ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [51ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [52ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [52ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [130ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [131ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [132ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [132ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [230ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [231ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [232ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [232ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [1227ms] refreshing cache from master, [1227ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1229ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@367096678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@a9cbdd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592) -> (continuation of Deferred@998565993 after retry RPC@1761113615) -> (continuation of Deferred@1693456267 after retry RPC@1326664980), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d17775b -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592) -> (continuation of Deferred@998565993 after retry RPC@1761113615) -> (continuation of Deferred@1693456267 after retry RPC@1326664980))) after lookup
18:21:39.648 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] connecting to peer
18:21:39.649 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:39.649 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:39.650 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735
18:21:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.774685 24748 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 7f6251917bba4996b6e74abedae69139)
18:21:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.774781 24748 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.775111 24748 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:39.775624 24452 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:39.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:39.776047 24452 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.779381 24689 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2" is_pre_election: true
18:21:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.779541 24689 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 9.
18:21:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.779794 24452 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 86740cf8056941e1bca1ebcce92945d2; no voters: 7f6251917bba4996b6e74abedae69139
18:21:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.779940 24748 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 10 FOLLOWER]: Leader pre-election won for term 11
18:21:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.779985 24748 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 7f6251917bba4996b6e74abedae69139)
18:21:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.780000 24748 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 10 FOLLOWER]: Advancing to term 11
18:21:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.780555 24748 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.780684 24748 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 11 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130:37215)
18:21:39.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.780951 24689 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "86740cf8056941e1bca1ebcce92945d2"
18:21:39.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.781056 24689 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 9 FOLLOWER]: Advancing to term 11
18:21:39.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:39.781179 24452 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111)
18:21:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.782050 24689 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f3eada3ec394bf9b1cc2e2a14c0e505 in term 11.
18:21:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.782264 24452 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6f3eada3ec394bf9b1cc2e2a14c0e505, 86740cf8056941e1bca1ebcce92945d2; no voters: 7f6251917bba4996b6e74abedae69139
18:21:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.782393 24748 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 11 FOLLOWER]: Leader election won for term 11
18:21:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.782542 24748 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 11 LEADER]: Becoming Leader. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Running, Role: LEADER
18:21:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.782630 24748 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:39.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:39.783363 22780 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 reported cstate change: term changed from 10 to 11, leader changed from 7f6251917bba4996b6e74abedae69139 (127.22.16.129) to 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131). New cstate: current_term: 11 leader_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: HEALTHY } } }
18:21:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:40.322477 24689 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 11 FOLLOWER]: Refusing update from remote peer 6f3eada3ec394bf9b1cc2e2a14c0e505: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 37. (index mismatch)
18:21:40.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:40.322820 24765 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
18:21:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:40.326530 24452 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 -> Peer 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735): Couldn't send request to peer 7f6251917bba4996b6e74abedae69139. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.129:41735: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:42.186 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:42.187 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:42.187 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=13, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=3769), Traces: [0ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1ms] delaying RPC due to: Illegal state: connection is terminated, [7ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [9ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [9ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [29ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [31ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [32ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [50ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [51ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [52ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [52ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [128ms] refreshing cache from master, [128ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [130ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [131ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [132ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [132ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [228ms] refreshing cache from master, [228ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [230ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [231ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [232ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [232ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [1227ms] refreshing cache from master, [1227ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1229ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [1231ms] sending RPC to server 7f6251917bba4996b6e74abedae69139, [1233ms] received response from server 7f6251917bba4996b6e74abedae69139: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [1233ms] delaying RPC due to: Network error: Failed to connect to peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735): Connection refused: /127.22.16.129:41735, [3767ms] refreshing cache from master, [3767ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [3769ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@367096678(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@a9cbdd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592) -> (continuation of Deferred@998565993 after retry RPC@1761113615) -> (continuation of Deferred@1693456267 after retry RPC@1326664980) -> (continuation of Deferred@1554168415 after retry RPC@151791410), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5d17775b -> wakeup thread Time-limited test -> (continuation of Deferred@1400854478 after retry RPC@892624725) -> (continuation of Deferred@713630627 after retry RPC@110802592) -> (continuation of Deferred@998565993 after retry RPC@1761113615) -> (continuation of Deferred@1693456267 after retry RPC@1326664980) -> (continuation of Deferred@1554168415 after retry RPC@151791410))) after lookup
18:21:42.188 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:42.189 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] Successfully connected to peer
18:21:42.215 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0xb18bf876, L:/127.0.0.1:54426 - R:/127.22.16.131:42931] using TOKEN/null
18:21:42.232 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.129:41735
18:21:42.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.232609 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:41735
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44715
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.308104 24788 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.308308 24788 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.308338 24788 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:42.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.310086 24788 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:42.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.310168 24788 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.129
18:21:42.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.311715 24788 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:42.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:42.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.129:41735
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.129
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44715
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24788
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.129
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/logs
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.311985 24788 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:42.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.312235 24788 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.314900 24794 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.314917 24793 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:42.314922 24796 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.315218 24788 server_base.cc:1061] running on GCE node
18:21:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.315392 24788 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.315582 24788 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.316747 24788 hybrid_clock.cc:648] HybridClock initialized: now 1783534902316727 us; error 32 us; skew 500 ppm
18:21:42.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.317793 24788 webserver.cc:534] Webserver started at http://127.22.16.129:44715/ using document root <none> and password file <none>
18:21:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.317981 24788 fs_manager.cc:362] Metadata directory not provided
18:21:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.318013 24788 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.319017 24788 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.001s
18:21:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.319623 24802 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.319761 24788 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.000s
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.319802 24788 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f6251917bba4996b6e74abedae69139"
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:11 on dist-test-slave-v72b"
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.320043 24788 fs_report.cc:389] FS layout report
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/wal
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/data
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.336994 24788 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.337246 24788 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.337327 24788 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.337500 24788 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:42.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.337998 24809 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:42.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.338763 24788 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:42.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.338833 24788 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.338868 24788 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.339561 24788 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.339613 24788 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
18:21:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.339643 24809 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap starting.
18:21:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.346107 24788 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.129:41735
18:21:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.346158 24916 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.129:41735 every 8 connection(s)
18:21:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.346448 24788 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-0/data/info.pb
18:21:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.347426 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 24788
18:21:42.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:42.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.353528 24809 log.cc:826] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Log is configured to *not* fsync() on all Append() calls
18:21:42.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.356865 24809 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:42.358 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:42.359 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.22.16.131:42931
18:21:42.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.357201 24809 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Bootstrap complete.
18:21:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.358495 24809 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent bootstrapping tablet: real 0.019s	user 0.004s	sys 0.010s
18:21:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.359635 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 24437
18:21:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.359679 24809 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.359959 24809 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f6251917bba4996b6e74abedae69139, State: Initialized, Role: FOLLOWER
18:21:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.360172 24809 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.360248 24917 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.360332 24917 heartbeater.cc:461] Registering TS with master...
18:21:42.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.360425 24809 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
18:21:42.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.360526 24917 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:42.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.361335 22636 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:42.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.362275 24918 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:42.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.362339 24918 heartbeater.cc:461] Registering TS with master...
18:21:42.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.362448 24918 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.362927 22780 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.363124 24919 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.363178 24919 heartbeater.cc:461] Registering TS with master...
18:21:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.363293 24919 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.363492 22780 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.129:38383
18:21:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.363587 22708 ts_manager.cc:194] Re-registered known tserver with Master: 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735)
18:21:42.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:42.364125 24918 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:42.366 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] handling channelInactive
18:21:42.367 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state READY due to: connection closed
18:21:42.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:42.368 [DEBUG - Time-limited test] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:42.369 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.369 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.369 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.386 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:42.386 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:42.386 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [2ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1935654206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@1b8221d0 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@641ffa7f -> wakeup thread Time-limited test)) after lookup
18:21:42.387 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:42.388 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.388 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.388 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.406 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:42.407 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:42.407 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [2ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [20ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1935654206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@1b8221d0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@641ffa7f -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994))) after lookup
18:21:42.407 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:42.408 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.409 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.446 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:42.446 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:42.447 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [2ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [20ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [40ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [42ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1935654206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@1b8221d0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@641ffa7f -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928))) after lookup
18:21:42.447 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:42.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.448 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.567 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:42.567 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:42.567 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=200), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [2ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [20ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [40ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [42ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [80ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [81ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [81ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [197ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [199ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1935654206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@1b8221d0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928) -> (continuation of Deferred@1045609801 after retry RPC@206038825), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@641ffa7f -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928) -> (continuation of Deferred@1045609801 after retry RPC@206038825))) after lookup
18:21:42.568 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:42.569 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.569 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:42.569 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:43.047 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:43.047 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)[L],7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)]}]
18:21:43.047 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=680), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [2ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [20ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [40ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [42ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [80ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [81ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [81ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [197ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [199ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [201ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [202ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [202ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [677ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [679ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1935654206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@1b8221d0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928) -> (continuation of Deferred@1045609801 after retry RPC@206038825) -> (continuation of Deferred@1009290029 after retry RPC@258301230), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@641ffa7f -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928) -> (continuation of Deferred@1045609801 after retry RPC@206038825) -> (continuation of Deferred@1009290029 after retry RPC@258301230))) after lookup
18:21:43.048 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] connecting to peer
18:21:43.049 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:43.049 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:43.049 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931) for tablet f98f377903864cbda11858a1e6d986be: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931
18:21:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.732182 24930 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.732311 24930 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.732595 24930 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:43.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:43.733022 24623 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:43.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:43.733597 24623 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.736308 24871 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139" is_pre_election: true
18:21:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.736425 24871 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 10.
18:21:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.736652 24621 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.736830 24930 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 11 FOLLOWER]: Leader pre-election won for term 12
18:21:43.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.736900 24930 raft_consensus.cc:493] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 11 FOLLOWER]: Starting leader election (detected failure of leader 6f3eada3ec394bf9b1cc2e2a14c0e505)
18:21:43.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.736930 24930 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 11 FOLLOWER]: Advancing to term 12
18:21:43.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.737591 24930 raft_consensus.cc:515] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:43.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.737813 24930 leader_election.cc:290] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 12 election: Requested vote from peers 7f6251917bba4996b6e74abedae69139 (127.22.16.129:41735), 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:43.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.737905 24871 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f98f377903864cbda11858a1e6d986be" candidate_uuid: "86740cf8056941e1bca1ebcce92945d2" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "7f6251917bba4996b6e74abedae69139"
18:21:43.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.738036 24871 raft_consensus.cc:3060] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 10 FOLLOWER]: Advancing to term 12
18:21:43.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:43.738289 24623 leader_election.cc:336] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111)
18:21:43.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.739055 24871 raft_consensus.cc:2468] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 86740cf8056941e1bca1ebcce92945d2 in term 12.
18:21:43.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.739296 24621 leader_election.cc:304] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7f6251917bba4996b6e74abedae69139, 86740cf8056941e1bca1ebcce92945d2; no voters: 6f3eada3ec394bf9b1cc2e2a14c0e505
18:21:43.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.739393 24930 raft_consensus.cc:2804] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 12 FOLLOWER]: Leader election won for term 12
18:21:43.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.739579 24930 raft_consensus.cc:697] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [term 12 LEADER]: Becoming Leader. State: Replica: 86740cf8056941e1bca1ebcce92945d2, State: Running, Role: LEADER
18:21:43.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.739684 24930 consensus_queue.cc:237] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:43.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:43.740411 22780 catalog_manager.cc:5696] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 reported cstate change: term changed from 11 to 12, leader changed from 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131) to 86740cf8056941e1bca1ebcce92945d2 (127.22.16.130). New cstate: current_term: 12 leader_uuid: "86740cf8056941e1bca1ebcce92945d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } health_report { overall_health: UNKNOWN } } }
18:21:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.177906 24871 raft_consensus.cc:1275] T f98f377903864cbda11858a1e6d986be P 7f6251917bba4996b6e74abedae69139 [term 12 FOLLOWER]: Refusing update from remote peer 86740cf8056941e1bca1ebcce92945d2: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 41. (index mismatch)
18:21:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.178290 24930 consensus_queue.cc:1048] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
18:21:44.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.181532 24623 consensus_peers.cc:597] T f98f377903864cbda11858a1e6d986be P 86740cf8056941e1bca1ebcce92945d2 -> Peer 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931): Couldn't send request to peer 6f3eada3ec394bf9b1cc2e2a14c0e505. Status: Network error: Client connection negotiation failed: client connection to 127.22.16.131:42931: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:44.207 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f98f377903864cbda11858a1e6d986be for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432' with partition [<start>, <end>)
18:21:44.207 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=f98f377903864cbda11858a1e6d986be@[6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931),7f6251917bba4996b6e74abedae69139(127.22.16.129:41735),86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)[L]]}]
18:21:44.208 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=f98f377903864cbda11858a1e6d986be, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1840), Traces: [0ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [2ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [19ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [20ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [21ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [21ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [39ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [40ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [42ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [79ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [80ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [81ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [81ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [197ms] refreshing cache from master, [198ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [199ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [201ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [202ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [202ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [677ms] refreshing cache from master, [677ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [679ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, [681ms] sending RPC to server 6f3eada3ec394bf9b1cc2e2a14c0e505, [682ms] received response from server 6f3eada3ec394bf9b1cc2e2a14c0e505: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [682ms] delaying RPC due to: Network error: Failed to connect to peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931): Connection refused: /127.22.16.131:42931, [1838ms] refreshing cache from master, [1838ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.16.188:46685, [1840ms] Sub RPC GetTableLocations: received response from server master-127.22.16.188:46685: OK, deferred=Deferred@1935654206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007fbf5c36aeb8@1b8221d0 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928) -> (continuation of Deferred@1045609801 after retry RPC@206038825) -> (continuation of Deferred@1009290029 after retry RPC@258301230) -> (continuation of Deferred@2048583115 after retry RPC@1605589899), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@641ffa7f -> wakeup thread Time-limited test -> (continuation of Deferred@1846174886 after retry RPC@1969075994) -> (continuation of Deferred@26154476 after retry RPC@435328928) -> (continuation of Deferred@1045609801 after retry RPC@206038825) -> (continuation of Deferred@1009290029 after retry RPC@258301230) -> (continuation of Deferred@2048583115 after retry RPC@1605589899))) after lookup
18:21:44.208 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] connecting to peer
18:21:44.209 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] Successfully connected to peer
18:21:44.239 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x8ce4a2f7, L:/127.0.0.1:53446 - R:/127.22.16.130:37215] using TOKEN/null
18:21:44.252 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.22.16.131:42931
18:21:44.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.253294 22594 external_mini_cluster.cc:1372] Running /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93oYzr/build/release/bin/kudu
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.332046 24958 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.332233 24958 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.332265 24958 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:44.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.333819 24958 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:44.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.333901 24958 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.16.131
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.335508 24958 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.16.148:39565
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.16.131:42931
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.16.131
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41001
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.16.190:35499,127.22.16.189:39739,127.22.16.188:46685
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24958
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.16.131
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/logs
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3ddc7d4404ac90c834b1cdd020973c286611e9b0
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 18:01:16 UTC on e7f111948823
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16869
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.335738 24958 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.336007 24958 file_cache.cc:492] Constructed file cache file cache with capacity 419430
18:21:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.338876 24963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.338915 24966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 18:21:44.338956 24964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.339076 24958 server_base.cc:1061] running on GCE node
18:21:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.339217 24958 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.339474 24958 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:44.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.340626 24958 hybrid_clock.cc:648] HybridClock initialized: now 1783534904340612 us; error 24 us; skew 500 ppm
18:21:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.341665 24958 webserver.cc:534] Webserver started at http://127.22.16.131:41001/ using document root <none> and password file <none>
18:21:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.341929 24958 fs_manager.cc:362] Metadata directory not provided
18:21:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.342005 24958 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.343137 24958 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:21:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.343848 24972 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.344058 24958 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.000s
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.344137 24958 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data,/tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505"
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 18:21:12 on dist-test-slave-v72b"
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.344408 24958 fs_report.cc:389] FS layout report
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/wal
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/data
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.357731 24958 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.358067 24958 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.358209 24958 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.358425 24958 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:44.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.358897 24979 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
18:21:44.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.359778 24958 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:44.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.359865 24958 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
18:21:44.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.359903 24958 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:44.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.360455 24958 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:44.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.360510 24958 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:21:44.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.360541 24979 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap starting.
18:21:44.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.367756 24958 rpc_server.cc:307] RPC server started. Bound to: 127.22.16.131:42931
18:21:44.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.367826 25086 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.16.131:42931 every 8 connection(s)
18:21:44.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.368155 24958 server_base.cc:1193] Dumped server information to /tmp/dist-test-task93oYzr/test-tmp/mini-kudu-cluster10610932776920553162/ts-2/data/info.pb
18:21:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.370538 24979 log.cc:826] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Log is configured to *not* fsync() on all Append() calls
18:21:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375524 25088 heartbeater.cc:344] Connected to a master server at 127.22.16.188:46685
18:21:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375607 25087 heartbeater.cc:344] Connected to a master server at 127.22.16.190:35499
18:21:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375674 25087 heartbeater.cc:461] Registering TS with master...
18:21:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375650 25089 heartbeater.cc:344] Connected to a master server at 127.22.16.189:39739
18:21:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375730 25089 heartbeater.cc:461] Registering TS with master...
18:21:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375658 24979 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375875 25089 heartbeater.cc:507] Master 127.22.16.189:39739 requested a full tablet report, sending...
18:21:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375875 25087 heartbeater.cc:507] Master 127.22.16.190:35499 requested a full tablet report, sending...
18:21:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.376070 24979 tablet_bootstrap.cc:492] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Bootstrap complete.
18:21:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.376446 22708 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.376487 22636 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:44.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.377154 24979 ts_tablet_manager.cc:1403] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent bootstrapping tablet: real 0.017s	user 0.007s	sys 0.007s
18:21:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.377923 24979 raft_consensus.cc:359] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.378098 24979 raft_consensus.cc:740] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f3eada3ec394bf9b1cc2e2a14c0e505, State: Initialized, Role: FOLLOWER
18:21:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.378196 24979 consensus_queue.cc:260] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f6251917bba4996b6e74abedae69139" member_type: VOTER last_known_addr { host: "127.22.16.129" port: 41735 } } peers { permanent_uuid: "86740cf8056941e1bca1ebcce92945d2" member_type: VOTER last_known_addr { host: "127.22.16.130" port: 37215 } } peers { permanent_uuid: "6f3eada3ec394bf9b1cc2e2a14c0e505" member_type: VOTER last_known_addr { host: "127.22.16.131" port: 42931 } }
18:21:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.378364 22594 external_mini_cluster.cc:1434] Started /tmp/dist-test-task93oYzr/build/release/bin/kudu as pid 24958
18:21:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.378461 24979 ts_tablet_manager.cc:1434] T f98f377903864cbda11858a1e6d986be P 6f3eada3ec394bf9b1cc2e2a14c0e505: Time spent starting tablet: real 0.001s	user 0.002s	sys 0.000s
18:21:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.375624 25088 heartbeater.cc:461] Registering TS with master...
18:21:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.378612 25088 heartbeater.cc:507] Master 127.22.16.188:46685 requested a full tablet report, sending...
18:21:44.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:44.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.378924 22781 ts_manager.cc:194] Re-registered known tserver with Master: 6f3eada3ec394bf9b1cc2e2a14c0e505 (127.22.16.131:42931)
18:21:44.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.379343 22781 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.16.131:36607
18:21:44.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.379973 25088 heartbeater.cc:499] Master 127.22.16.188:46685 was elected leader, sending a full tablet report...
18:21:44.382 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:44.384 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783534872432, tablet=f98f377903864cbda11858a1e6d986be, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:44.389 [DEBUG - main] (Connection.java:575) [peer 6f3eada3ec394bf9b1cc2e2a14c0e505(127.22.16.131:42931)] disconnecting while in state TERMINATED
18:21:44.391 [DEBUG - main] (Connection.java:575) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] disconnecting while in state READY
18:21:44.391 [DEBUG - main] (Connection.java:575) [peer master-127.22.16.189:39739(127.22.16.189:39739)] disconnecting while in state READY
18:21:44.391 [DEBUG - main] (Connection.java:575) [peer 7f6251917bba4996b6e74abedae69139(127.22.16.129:41735)] disconnecting while in state TERMINATED
18:21:44.391 [DEBUG - main] (Connection.java:575) [peer master-127.22.16.190:35499(127.22.16.190:35499)] disconnecting while in state READY
18:21:44.391 [DEBUG - main] (Connection.java:575) [peer master-127.22.16.188:46685(127.22.16.188:46685)] disconnecting while in state READY
18:21:44.394 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.16.188:46685(127.22.16.188:46685)] handling channelInactive
18:21:44.394 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.16.188:46685(127.22.16.188:46685)] cleaning up while in state READY due to: connection closed
18:21:44.394 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] handling channelInactive
18:21:44.394 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 86740cf8056941e1bca1ebcce92945d2(127.22.16.130:37215)] cleaning up while in state READY due to: connection closed
18:21:44.395 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.16.190:35499(127.22.16.190:35499)] handling channelInactive
18:21:44.395 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.16.190:35499(127.22.16.190:35499)] cleaning up while in state READY due to: connection closed
18:21:44.396 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:21:44.397 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.22.16.189:39739(127.22.16.189:39739)] handling channelInactive
18:21:44.397 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.22.16.189:39739(127.22.16.189:39739)] cleaning up while in state READY due to: connection closed
18:21:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.397753 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 24788
18:21:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.404639 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 24606
18:21:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.410610 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 24958
18:21:44.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.415580 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 22605
18:21:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.420941 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 22677
18:21:44.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 18:21:44.426599 22594 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task93oYzr/build/release/bin/kudu with pid 22747
18:21:44.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T18:21:44Z chronyd exiting

Time: 34.205

OK (1 test)

