JUnit version 4.13.2
.21:07:54.903 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783544874893
21:07:54.974 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
21:07:55.014 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:07:55.029 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDb8yoT/test-tmp
21:07:55.053 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
21:07:55.056 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDb8yoT/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
21:07:55.248 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:07:55.261 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:07:55.262 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:07:55.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.263243 16685 mini_oidc.cc:150] Starting JWKS server
21:07:55.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.263922 16685 webserver.cc:534] Webserver started at http://127.0.0.1:33499/ using document root <none> and password file <none>
21:07:55.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.264016 16685 mini_oidc.cc:186] Starting OIDC Discovery server
21:07:55.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.264180 16685 webserver.cc:534] Webserver started at http://127.0.0.1:46851/ using document root <none> and password file <none>
21:07:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:07:55Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:07:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:07:55Z Disabled control of system clock
21:07:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.276742 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/wal
21:07:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data
21:07:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/logs
21:07:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data/info.pb
21:07:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:55.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:55.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:55.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.126:34341
21:07:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.126
21:07:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33499/jwks.json with env {}
21:07:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.359755 16696 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:07:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.360019 16696 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.360054 16696 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:07:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.361995 16696 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:07:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.362076 16696 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:07:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.362099 16696 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.362118 16696 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:07:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.362138 16696 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.362157 16696 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33499/jwks.json
21:07:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.364425 16696 master_runner.cc:387] Master server non-default flags:
21:07:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:55.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data
21:07:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/wal
21:07:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.126:34341
21:07:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:55.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:55.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:55.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33499/jwks.json
21:07:55.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data/info.pb
21:07:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.126
21:07:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.16696
21:07:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/logs
21:07:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.364734 16696 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.365007 16696 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.367820 16696 server_base.cc:1061] running on GCE node
21:07:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.367928 16702 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.367976 16701 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.368152 16704 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.368342 16696 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.368598 16696 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.369741 16696 hybrid_clock.cc:648] HybridClock initialized: now 1783544875369719 us; error 38 us; skew 500 ppm
21:07:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.371068 16696 webserver.cc:534] Webserver started at http://127.16.75.126:44299/ using document root <none> and password file <none>
21:07:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.371233 16696 fs_manager.cc:362] Metadata directory not provided
21:07:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.371268 16696 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.371340 16696 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:07:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.372404 16696 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data/instance:
21:07:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae32555cb4894c36ad192ab1b65d1435"
21:07:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.372802 16696 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/wal/instance:
21:07:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae32555cb4894c36ad192ab1b65d1435"
21:07:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.374454 16696 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.001s	sys 0.001s
21:07:55.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.376821 16710 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:07:55.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.377033 16696 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
21:07:55.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.377103 16696 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/wal
21:07:55.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae32555cb4894c36ad192ab1b65d1435"
21:07:55.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.377179 16696 fs_report.cc:389] FS layout report
21:07:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/wal
21:07:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/wal
21:07:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data/data
21:07:55.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:55.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:55.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:55.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.406230 16696 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:07:55.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.406543 16696 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:55.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.406631 16696 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.410667 16696 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.126:34341
21:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.410741 16762 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.126:34341 every 8 connection(s)
21:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.411006 16696 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/data/info.pb
21:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.411708 16763 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.413137 16763 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.414877 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 16696
21:07:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.414999 16685 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-0/wal/instance
21:07:55.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.416023 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/wal
21:07:55.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data
21:07:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/logs
21:07:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data/info.pb
21:07:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.125:36119
21:07:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.125
21:07:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:55.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:55.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:55.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33499/jwks.json with env {}
21:07:55.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.420413 16763 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.420936 16714 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.125:36119: connect: Connection refused (error 111)
21:07:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.421505 16763 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.75.125:36119: Network error: Client connection negotiation failed: client connection to 127.16.75.125:36119: connect: Connection refused (error 111)
21:07:55.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.470719 16763 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } attempt: 1
21:07:55.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.471714 16763 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.75.125:36119: Network error: Client connection negotiation failed: client connection to 127.16.75.125:36119: connect: Connection refused (error 111)
21:07:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.498519 16767 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:07:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.498728 16767 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.498749 16767 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:07:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.500722 16767 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:07:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.500779 16767 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:07:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.500794 16767 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.500806 16767 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:07:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.500819 16767 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.500830 16767 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33499/jwks.json
21:07:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.503328 16767 master_runner.cc:387] Master server non-default flags:
21:07:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data
21:07:55.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/wal
21:07:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.125:36119
21:07:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33499/jwks.json
21:07:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data/info.pb
21:07:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.125
21:07:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.16767
21:07:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/logs
21:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.503646 16767 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.503929 16767 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.506596 16776 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.506780 16774 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.506949 16773 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.507579 16767 server_base.cc:1061] running on GCE node
21:07:55.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.507764 16767 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:55.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.508046 16767 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:55.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.509193 16767 hybrid_clock.cc:648] HybridClock initialized: now 1783544875509176 us; error 30 us; skew 500 ppm
21:07:55.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.510324 16767 webserver.cc:534] Webserver started at http://127.16.75.125:33217/ using document root <none> and password file <none>
21:07:55.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.510542 16767 fs_manager.cc:362] Metadata directory not provided
21:07:55.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.510598 16767 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:55.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.510704 16767 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:07:55.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.511889 16767 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data/instance:
21:07:55.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "421e629960e949b28f021a32f47e6e6a"
21:07:55.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.512296 16767 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/wal/instance:
21:07:55.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "421e629960e949b28f021a32f47e6e6a"
21:07:55.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.514007 16767 fs_manager.cc:696] Time spent creating directory manager: real 0.002s	user 0.002s	sys 0.000s
21:07:55.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.514919 16782 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:07:55.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.515095 16767 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:07:55.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.515154 16767 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/wal
21:07:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "421e629960e949b28f021a32f47e6e6a"
21:07:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.515223 16767 fs_report.cc:389] FS layout report
21:07:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/wal
21:07:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/wal
21:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data/data
21:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:55.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.539930 16763 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } attempt: 2
21:07:55.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.540462 16767 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:07:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.540686 16763 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.75.125:36119: Network error: Client connection negotiation failed: client connection to 127.16.75.125:36119: connect: Connection refused (error 111)
21:07:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.540817 16767 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.540949 16767 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.545944 16767 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.125:36119
21:07:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.545976 16834 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.125:36119 every 8 connection(s)
21:07:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.546286 16767 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/data/info.pb
21:07:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.546855 16835 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.548158 16835 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.551393 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 16767
21:07:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.551486 16685 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-1/wal/instance
21:07:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.551956 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/wal
21:07:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data
21:07:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/logs
21:07:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data/info.pb
21:07:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.124:37917
21:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.124
21:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33499/jwks.json with env {}
21:07:55.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.553612 16835 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.559202 16835 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.560074 16783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.124:37917: connect: Connection refused (error 111)
21:07:55.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.560937 16835 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.75.124:37917: Network error: Client connection negotiation failed: client connection to 127.16.75.124:37917: connect: Connection refused (error 111)
21:07:55.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.610173 16835 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } attempt: 1
21:07:55.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.611069 16835 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.75.124:37917: Network error: Client connection negotiation failed: client connection to 127.16.75.124:37917: connect: Connection refused (error 111)
21:07:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.631867 16763 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } attempt: 3
21:07:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.632454 16838 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:07:55.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.632684 16838 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:55.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.632723 16838 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:07:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.635036 16838 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:07:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.635325 16763 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.635720 16838 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:07:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.635777 16838 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.635800 16838 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:07:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.635821 16838 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.635886 16838 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33499/jwks.json
21:07:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.636124 16763 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.75.124:37917: Network error: Client connection negotiation failed: client connection to 127.16.75.124:37917: connect: Connection refused (error 111)
21:07:55.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.637693 16838 master_runner.cc:387] Master server non-default flags:
21:07:55.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/wal
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.124:37917
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33499/jwks.json
21:07:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data/info.pb
21:07:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.124
21:07:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.16838
21:07:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/logs
21:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.637914 16838 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.638103 16838 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.640337 16847 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.640445 16849 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.640332 16846 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.640607 16838 server_base.cc:1061] running on GCE node
21:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.640790 16838 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.641007 16838 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.642135 16838 hybrid_clock.cc:648] HybridClock initialized: now 1783544875642112 us; error 37 us; skew 500 ppm
21:07:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.643271 16838 webserver.cc:534] Webserver started at http://127.16.75.124:33727/ using document root <none> and password file <none>
21:07:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.643491 16838 fs_manager.cc:362] Metadata directory not provided
21:07:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.643553 16838 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:55.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.643651 16838 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:07:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.644743 16838 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data/instance:
21:07:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74374be7c187402ea618f4a07c8fe370"
21:07:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.645107 16838 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/wal/instance:
21:07:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74374be7c187402ea618f4a07c8fe370"
21:07:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.646616 16838 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.002s	sys 0.000s
21:07:55.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.647462 16855 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:07:55.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.647627 16838 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.647681 16838 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/wal
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74374be7c187402ea618f4a07c8fe370"
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.647739 16838 fs_report.cc:389] FS layout report
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/wal
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/wal
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data/data
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.667268 16763 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } attempt: 1
21:07:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.668052 16763 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.75.124:37917: Network error: Client connection negotiation failed: client connection to 127.16.75.124:37917: connect: Connection refused (error 111)
21:07:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.668694 16838 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:07:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.669032 16838 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.669195 16838 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.673614 16838 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.124:37917
21:07:55.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.673646 16907 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.124:37917 every 8 connection(s)
21:07:55.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.673971 16838 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/data/info.pb
21:07:55.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.675102 16908 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.676441 16908 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.677620 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 16838
21:07:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.677688 16685 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/master-2/wal/instance
21:07:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.679229 16835 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } attempt: 2
21:07:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.683550 16908 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.684773 16835 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a: Bootstrap starting.
21:07:55.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.685513 16835 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a: Neither blocks nor log segments found. Creating new log.
21:07:55.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.685788 16835 log.cc:826] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a: Log is configured to *not* fsync() on all Append() calls
21:07:55.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.686411 16835 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a: No bootstrap required, opened a new log
21:07:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.688062 16835 raft_consensus.cc:359] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.688181 16835 raft_consensus.cc:385] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.688211 16835 raft_consensus.cc:740] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 421e629960e949b28f021a32f47e6e6a, State: Initialized, Role: FOLLOWER
21:07:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.688311 16835 consensus_queue.cc:260] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.688638 16835 sys_catalog.cc:565] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: configured and running, proceeding with master startup.
21:07:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.690578 16908 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } has no permanent_uuid. Determining permanent_uuid...
21:07:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.690701 16767 master_runner.cc:187] Error getting master registration for 127.16.75.126:34341: OK, instance_id { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" instance_seqno: 1783544875409551 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:07:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.692066 16767 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:07:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.692087 16917 sys_catalog.cc:455] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.692181 16917 sys_catalog.cc:458] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: This master's current role is: FOLLOWER
21:07:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.693054 16928 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:07:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.693110 16928 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:07:55.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.694104 16908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: Bootstrap starting.
21:07:55.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.694666 16908 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: Neither blocks nor log segments found. Creating new log.
21:07:55.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.694926 16908 log.cc:826] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: Log is configured to *not* fsync() on all Append() calls
21:07:55.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.695591 16908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: No bootstrap required, opened a new log
21:07:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.697719 16908 raft_consensus.cc:359] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.697847 16908 raft_consensus.cc:385] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.697880 16908 raft_consensus.cc:740] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74374be7c187402ea618f4a07c8fe370, State: Initialized, Role: FOLLOWER
21:07:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.697965 16908 consensus_queue.cc:260] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.698644 16908 sys_catalog.cc:565] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [sys.catalog]: configured and running, proceeding with master startup.
21:07:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.698627 16931 sys_catalog.cc:455] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.698740 16931 sys_catalog.cc:458] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [sys.catalog]: This master's current role is: FOLLOWER
21:07:55.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.700593 16942 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:07:55.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.700974 16942 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:07:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.701318 16838 master_runner.cc:187] Error getting master registration for 127.16.75.126:34341: OK, instance_id { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" instance_seqno: 1783544875409551 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:07:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.702556 16838 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:07:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.743222 16763 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } attempt: 2
21:07:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.747989 16917 raft_consensus.cc:493] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.748093 16763 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: Bootstrap starting.
21:07:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.748124 16917 raft_consensus.cc:515] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.748432 16917 leader_election.cc:290] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ae32555cb4894c36ad192ab1b65d1435 (127.16.75.126:34341), 74374be7c187402ea618f4a07c8fe370 (127.16.75.124:37917)
21:07:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.748962 16763 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: Neither blocks nor log segments found. Creating new log.
21:07:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.749279 16763 log.cc:826] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: Log is configured to *not* fsync() on all Append() calls
21:07:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.750304 16763 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: No bootstrap required, opened a new log
21:07:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.750773 16882 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "421e629960e949b28f021a32f47e6e6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74374be7c187402ea618f4a07c8fe370" is_pre_election: true
21:07:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.750963 16882 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 421e629960e949b28f021a32f47e6e6a in term 0.
21:07:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.752504 16763 raft_consensus.cc:359] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.752669 16763 raft_consensus.cc:385] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.752712 16763 raft_consensus.cc:740] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ae32555cb4894c36ad192ab1b65d1435, State: Initialized, Role: FOLLOWER
21:07:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.752851 16763 consensus_queue.cc:260] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.753247 16763 sys_catalog.cc:565] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [sys.catalog]: configured and running, proceeding with master startup.
21:07:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.753325 16737 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "421e629960e949b28f021a32f47e6e6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae32555cb4894c36ad192ab1b65d1435" is_pre_election: true
21:07:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.753448 16737 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 421e629960e949b28f021a32f47e6e6a in term 0.
21:07:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.753768 16785 leader_election.cc:304] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 421e629960e949b28f021a32f47e6e6a, ae32555cb4894c36ad192ab1b65d1435; no voters: 
21:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.753930 16917 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Leader pre-election won for term 1
21:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.753986 16917 raft_consensus.cc:493] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.754019 16917 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 0 FOLLOWER]: Advancing to term 1
21:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.755064 16917 raft_consensus.cc:515] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.755216 16917 leader_election.cc:290] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [CANDIDATE]: Term 1 election: Requested vote from peers ae32555cb4894c36ad192ab1b65d1435 (127.16.75.126:34341), 74374be7c187402ea618f4a07c8fe370 (127.16.75.124:37917)
21:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.755484 16737 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "421e629960e949b28f021a32f47e6e6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae32555cb4894c36ad192ab1b65d1435"
21:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.755553 16737 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [term 0 FOLLOWER]: Advancing to term 1
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.755815 16882 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "421e629960e949b28f021a32f47e6e6a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74374be7c187402ea618f4a07c8fe370"
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.755931 16882 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [term 0 FOLLOWER]: Advancing to term 1
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.756486 16737 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 421e629960e949b28f021a32f47e6e6a in term 1.
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.756686 16785 leader_election.cc:304] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 421e629960e949b28f021a32f47e6e6a, ae32555cb4894c36ad192ab1b65d1435; no voters: 
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.756799 16917 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 1 FOLLOWER]: Leader election won for term 1
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.756852 16882 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 421e629960e949b28f021a32f47e6e6a in term 1.
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.756945 16917 raft_consensus.cc:697] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [term 1 LEADER]: Becoming Leader. State: Replica: 421e629960e949b28f021a32f47e6e6a, State: Running, Role: LEADER
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.757036 16917 consensus_queue.cc:237] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } }
21:07:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.757458 16917 sys_catalog.cc:455] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 421e629960e949b28f021a32f47e6e6a. Latest consensus state: current_term: 1 leader_uuid: "421e629960e949b28f021a32f47e6e6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.757525 16917 sys_catalog.cc:458] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: This master's current role is: LEADER
21:07:55.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.762535 16945 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.763029 16945 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [sys.catalog]: This master's current role is: FOLLOWER
21:07:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.764323 16956 catalog_manager.cc:1593] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:07:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.764639 16956 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:07:55.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.776085 16959 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
21:07:55.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.776950 16959 catalog_manager.cc:1519] Initializing Kudu cluster ID...
21:07:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.788957 16882 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [term 1 FOLLOWER]: Refusing update from remote peer 421e629960e949b28f021a32f47e6e6a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:07:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.788957 16737 raft_consensus.cc:1275] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [term 1 FOLLOWER]: Refusing update from remote peer 421e629960e949b28f021a32f47e6e6a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:07:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.789364 16957 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [LEADER]: Connected to new peer: Peer: permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:07:55.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.790525 16957 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [LEADER]: Connected to new peer: Peer: permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:07:55.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.791129 16945 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 421e629960e949b28f021a32f47e6e6a. Latest consensus state: current_term: 1 leader_uuid: "421e629960e949b28f021a32f47e6e6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.791218 16945 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [sys.catalog]: This master's current role is: FOLLOWER
21:07:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.791571 16788 tablet.cc:2406] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:07:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.792501 16931 sys_catalog.cc:455] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 421e629960e949b28f021a32f47e6e6a. Latest consensus state: current_term: 1 leader_uuid: "421e629960e949b28f021a32f47e6e6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.792606 16931 sys_catalog.cc:458] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [sys.catalog]: This master's current role is: FOLLOWER
21:07:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.801378 16957 sys_catalog.cc:455] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "421e629960e949b28f021a32f47e6e6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.801525 16957 sys_catalog.cc:458] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: This master's current role is: LEADER
21:07:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.803093 16931 sys_catalog.cc:455] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "421e629960e949b28f021a32f47e6e6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.803210 16931 sys_catalog.cc:458] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370 [sys.catalog]: This master's current role is: FOLLOWER
21:07:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.804101 16961 mvcc.cc:204] Tried to move back new op lower bound from 7305399811228643328 to 7305399811101536256. Current Snapshot: MvccSnapshot[applied={T|T < 7305399811228643328}]
21:07:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.804507 16964 mvcc.cc:204] Tried to move back new op lower bound from 7305399811228643328 to 7305399811101536256. Current Snapshot: MvccSnapshot[applied={T|T < 7305399811228643328}]
21:07:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.805938 16917 sys_catalog.cc:455] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "421e629960e949b28f021a32f47e6e6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.806054 16917 sys_catalog.cc:458] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a [sys.catalog]: This master's current role is: LEADER
21:07:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.806649 16945 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "421e629960e949b28f021a32f47e6e6a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ae32555cb4894c36ad192ab1b65d1435" member_type: VOTER last_known_addr { host: "127.16.75.126" port: 34341 } } peers { permanent_uuid: "421e629960e949b28f021a32f47e6e6a" member_type: VOTER last_known_addr { host: "127.16.75.125" port: 36119 } } peers { permanent_uuid: "74374be7c187402ea618f4a07c8fe370" member_type: VOTER last_known_addr { host: "127.16.75.124" port: 37917 } } }
21:07:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.806763 16945 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435 [sys.catalog]: This master's current role is: FOLLOWER
21:07:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.808941 16962 mvcc.cc:204] Tried to move back new op lower bound from 7305399811228643328 to 7305399811101536256. Current Snapshot: MvccSnapshot[applied={T|T < 7305399811228643328 or (T in {7305399811228643328})}]
21:07:55.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.809976 16959 catalog_manager.cc:1382] Generated new cluster ID: e7c412f1ac3840e4bf1e3ef0c41f6bc0
21:07:55.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.810043 16959 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
21:07:55.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.825532 16959 catalog_manager.cc:1405] Generated new certificate authority record
21:07:55.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.826923 16959 catalog_manager.cc:1539] Loading token signing keys...
21:07:55.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.859464 16959 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 421e629960e949b28f021a32f47e6e6a: Generated new TSK 0
21:07:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.861275 16959 catalog_manager.cc:1549] Initializing in-progress tserver states...
21:07:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.880787 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:07:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:0
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:07:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:07:55.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.990511 16969 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:07:55.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.990700 16969 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:55.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.990721 16969 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:07:55.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.992398 16969 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:07:55.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.992472 16969 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.65
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.994107 16969 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:0
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.16969
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.994331 16969 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.994539 16969 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.997452 16975 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.997679 16974 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:55.998201 16977 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.998670 16969 server_base.cc:1061] running on GCE node
21:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.998919 16969 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:55.999150 16969 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:56.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.000309 16969 hybrid_clock.cc:648] HybridClock initialized: now 1783544876000286 us; error 43 us; skew 500 ppm
21:07:56.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.001509 16969 webserver.cc:534] Webserver started at http://127.16.75.65:46433/ using document root <none> and password file <none>
21:07:56.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.001751 16969 fs_manager.cc:362] Metadata directory not provided
21:07:56.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.001817 16969 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:56.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.001935 16969 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:07:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.002884 16969 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/instance:
21:07:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:07:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.003274 16969 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal/instance:
21:07:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:07:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.004814 16969 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.003s
21:07:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.005748 16983 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.005955 16969 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.006026 16969 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.006093 16969 fs_report.cc:389] FS layout report
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/data
21:07:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.014079 16969 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:07:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.014508 16969 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.014647 16969 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.014919 16969 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.015365 16969 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.015450 16969 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:07:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.015487 16969 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.015506 16969 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:07:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.022173 16969 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.65:37515
21:07:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.022219 17096 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.65:37515 every 8 connection(s)
21:07:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.022629 16969 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:07:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.029544 17098 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:07:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.029675 17098 heartbeater.cc:461] Registering TS with master...
21:07:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.029737 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 16969
21:07:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.029856 16685 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal/instance
21:07:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.029942 17098 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:07:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.031147 17099 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:07:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.031229 17099 heartbeater.cc:461] Registering TS with master...
21:07:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.031342 17099 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:07:56.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.032166 16727 ts_manager.cc:194] Registered new tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:07:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.033144 16799 ts_manager.cc:194] Registered new tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:07:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.033203 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:07:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:0
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:56.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:07:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.034217 16799 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.65:43271
21:07:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.037578 17097 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:07:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.037670 17097 heartbeater.cc:461] Registering TS with master...
21:07:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.037757 17097 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:07:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.038201 16872 ts_manager.cc:194] Registered new tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:07:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.115434 17104 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.115624 17104 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.115644 17104 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:07:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.117254 17104 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:07:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.117321 17104 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.66
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.118947 17104 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:0
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:07:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17104
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:07:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:07:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:07:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:07:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.119156 17104 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.119339 17104 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.121781 17109 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.121781 17110 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.121814 17112 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.122357 17104 server_base.cc:1061] running on GCE node
21:07:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.122524 17104 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.122761 17104 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:56.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.123896 17104 hybrid_clock.cc:648] HybridClock initialized: now 1783544876123868 us; error 42 us; skew 500 ppm
21:07:56.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.125039 17104 webserver.cc:534] Webserver started at http://127.16.75.66:39729/ using document root <none> and password file <none>
21:07:56.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.125257 17104 fs_manager.cc:362] Metadata directory not provided
21:07:56.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.125325 17104 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:56.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.125447 17104 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:07:56.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.126322 17104 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/instance:
21:07:56.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c44178f4b5464e1aa2934204708c52a1"
21:07:56.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:07:56.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.126680 17104 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal/instance:
21:07:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c44178f4b5464e1aa2934204708c52a1"
21:07:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:07:56.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.128118 17104 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.002s	sys 0.001s
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.128885 17118 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.129069 17104 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.001s	sys 0.000s
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.129133 17104 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c44178f4b5464e1aa2934204708c52a1"
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.129194 17104 fs_report.cc:389] FS layout report
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:07:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/data
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.146593 17104 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:07:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.146993 17104 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.147135 17104 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.147396 17104 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.147898 17104 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.147967 17104 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:07:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.148002 17104 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.148020 17104 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:07:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.154809 17104 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.66:44849
21:07:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.154857 17231 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.66:44849 every 8 connection(s)
21:07:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.155267 17104 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:07:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.159478 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 17104
21:07:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.159585 16685 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal/instance
21:07:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.161247 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:07:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:07:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:07:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:0
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.161665 17234 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.161784 17234 heartbeater.cc:461] Registering TS with master...
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.161867 17232 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.161945 17232 heartbeater.cc:461] Registering TS with master...
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.162015 17234 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:07:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.162053 17232 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:07:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.162379 16799 ts_manager.cc:194] Registered new tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:07:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.162438 16872 ts_manager.cc:194] Registered new tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:07:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.162935 16799 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.66:46111
21:07:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.163098 17233 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:07:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.163148 17233 heartbeater.cc:461] Registering TS with master...
21:07:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.163257 17233 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:07:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.163579 16726 ts_manager.cc:194] Registered new tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:07:56.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.243660 17239 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:07:56.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.243916 17239 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:56.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.243940 17239 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:07:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.245482 17239 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:07:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.245540 17239 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.67
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.247100 17239 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:0
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17239
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:07:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:07:56.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.247318 17239 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:56.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.247517 17239 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.250012 17247 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.250025 17244 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:56.250283 17245 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.250520 17239 server_base.cc:1061] running on GCE node
21:07:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.250705 17239 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.250985 17239 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.252147 17239 hybrid_clock.cc:648] HybridClock initialized: now 1783544876252127 us; error 37 us; skew 500 ppm
21:07:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.253470 17239 webserver.cc:534] Webserver started at http://127.16.75.67:39433/ using document root <none> and password file <none>
21:07:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.253741 17239 fs_manager.cc:362] Metadata directory not provided
21:07:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.253811 17239 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.253925 17239 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:07:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.254968 17239 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/instance:
21:07:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:07:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:07:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.255336 17239 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal/instance:
21:07:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:07:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:07:56.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.256863 17239 fs_manager.cc:696] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.003s
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.257910 17253 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.258117 17239 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.258198 17239 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.258267 17239 fs_report.cc:389] FS layout report
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:07:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/data
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.276674 17239 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:07:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.277036 17239 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.277159 17239 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.277392 17239 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.277796 17239 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.277850 17239 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:07:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.277886 17239 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:56.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.277904 17239 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:07:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.284324 17239 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.67:41157
21:07:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.284366 17366 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.67:41157 every 8 connection(s)
21:07:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.284685 17239 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:07:56.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.286584 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 17239
21:07:56.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.286671 16685 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal/instance
21:07:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.290335 17367 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:07:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.290668 17369 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.290838 17367 heartbeater.cc:461] Registering TS with master...
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.290855 17369 heartbeater.cc:461] Registering TS with master...
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.290971 17368 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.291010 17368 heartbeater.cc:461] Registering TS with master...
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.291093 17368 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.291095 17367 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.291095 17369 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.291581 16726 ts_manager.cc:194] Registered new tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.291597 16872 ts_manager.cc:194] Registered new tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:07:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.291741 16799 ts_manager.cc:194] Registered new tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:07:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.292207 16799 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.67:39939
21:07:56.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.301469 16685 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
21:07:56.302 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:07:56.303 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

21:07:56.312 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.16.75.126"
      port: 34341
    }
    bound_http_address {
      host: "127.16.75.126"
      port: 44299
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.16.75.125"
      port: 36119
    }
    bound_http_address {
      host: "127.16.75.125"
      port: 33217
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.16.75.124"
      port: 37917
    }
    bound_http_address {
      host: "127.16.75.124"
      port: 33727
    }
  }
}

21:07:56.316 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:07:56.318 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.16.75.65"
      port: 37515
    }
    bound_http_address {
      host: "127.16.75.65"
      port: 46433
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.16.75.66"
      port: 44849
    }
    bound_http_address {
      host: "127.16.75.66"
      port: 39729
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.16.75.67"
      port: 41157
    }
    bound_http_address {
      host: "127.16.75.67"
      port: 39433
    }
  }
}

21:07:56.318 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:07:56.530 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:07:56.596 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
21:07:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.701719 16942 catalog_manager.cc:1294] Loaded cluster ID: e7c412f1ac3840e4bf1e3ef0c41f6bc0
21:07:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.701800 16942 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: loading cluster ID for follower catalog manager: success
21:07:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.703176 16942 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: acquiring CA information for follower catalog manager: success
21:07:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.703527 16942 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 74374be7c187402ea618f4a07c8fe370: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.765370 16956 catalog_manager.cc:1294] Loaded cluster ID: e7c412f1ac3840e4bf1e3ef0c41f6bc0
21:07:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.765455 16956 catalog_manager.cc:1587] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: loading cluster ID for follower catalog manager: success
21:07:56.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.766830 16956 catalog_manager.cc:1609] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: acquiring CA information for follower catalog manager: success
21:07:56.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:56.767177 16956 catalog_manager.cc:1637] T 00000000000000000000000000000000 P ae32555cb4894c36ad192ab1b65d1435: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:56.775 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.75.126:34341(127.16.75.126:34341)] connecting to peer
21:07:56.832 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.75.125:36119(127.16.75.125:36119)] connecting to peer
21:07:56.833 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.75.124:37917(127.16.75.124:37917)] connecting to peer
21:07:56.851 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.75.125:36119(127.16.75.125:36119)] Successfully connected to peer
21:07:56.851 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.75.126:34341(127.16.75.126:34341)] Successfully connected to peer
21:07:56.851 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.16.75.124:37917(127.16.75.124:37917)] Successfully connected to peer
21:07:56.945 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.75.125
21:07:56.948 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.75.126
21:07:56.948 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.75.124
21:07:57.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.035575 17099 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:07:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.163875 17234 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:07:57.172 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x88af1f3e, L:/127.0.0.1:51648 - R:/127.16.75.124:37917] using SASL/PLAIN
21:07:57.172 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8fff3e18, L:/127.0.0.1:39540 - R:/127.16.75.126:34341] using SASL/PLAIN
21:07:57.172 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7327a27d, L:/127.0.0.1:38554 - R:/127.16.75.125:36119] using SASL/PLAIN
21:07:57.209 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:07:57.210 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=477), Traces: [0ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.75.126:34341, [94ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.75.125:36119, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.75.124:37917, [449ms] Sub RPC ConnectToMaster: received response from server master-127.16.75.126:34341: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.16.75.124:37917: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.16.75.125:36119: OK, deferred=null) after lookup
21:07:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.256876 16799 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38554:
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718"
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
21:07:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
21:07:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
21:07:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:07:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:57.257231 16799 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:07:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.276371 17301 tablet_service.cc:1511] Processing CreateTablet for tablet 51618a18fd6f49398ed01f49b19c347f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718 [id=dec758c7663d415a98443ac02d75fd83]), partition=RANGE (key) PARTITION UNBOUNDED
21:07:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.276386 17031 tablet_service.cc:1511] Processing CreateTablet for tablet 51618a18fd6f49398ed01f49b19c347f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718 [id=dec758c7663d415a98443ac02d75fd83]), partition=RANGE (key) PARTITION UNBOUNDED
21:07:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.276710 17031 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51618a18fd6f49398ed01f49b19c347f. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.276710 17301 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51618a18fd6f49398ed01f49b19c347f. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.280014 17166 tablet_service.cc:1511] Processing CreateTablet for tablet 51618a18fd6f49398ed01f49b19c347f (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718 [id=dec758c7663d415a98443ac02d75fd83]), partition=RANGE (key) PARTITION UNBOUNDED
21:07:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.280297 17166 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51618a18fd6f49398ed01f49b19c347f. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.280452 17400 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap starting.
21:07:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.281105 17400 tablet_bootstrap.cc:654] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Neither blocks nor log segments found. Creating new log.
21:07:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.281400 17400 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Log is configured to *not* fsync() on all Append() calls
21:07:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.281697 17399 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap starting.
21:07:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.282572 17399 tablet_bootstrap.cc:654] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Neither blocks nor log segments found. Creating new log.
21:07:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.283143 17399 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Log is configured to *not* fsync() on all Append() calls
21:07:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.283963 17399 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: No bootstrap required, opened a new log
21:07:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.284056 17399 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent bootstrapping tablet: real 0.002s	user 0.001s	sys 0.000s
21:07:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.287492 17399 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.287650 17399 raft_consensus.cc:385] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.287678 17399 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Initialized, Role: FOLLOWER
21:07:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.287772 17399 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.288486 17399 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent starting tablet: real 0.004s	user 0.001s	sys 0.002s
21:07:57.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.290210 17369 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:07:57.294 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table dec758c7663d415a98443ac02d75fd83
21:07:57.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.296213 17402 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap starting.
21:07:57.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.296589 17400 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: No bootstrap required, opened a new log
21:07:57.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.296682 17400 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent bootstrapping tablet: real 0.016s	user 0.001s	sys 0.000s
21:07:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.298550 17400 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.298735 17400 raft_consensus.cc:385] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.298776 17400 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62a04151dfcd4eaf9bc55e4985fac600, State: Initialized, Role: FOLLOWER
21:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.298889 17402 tablet_bootstrap.cc:654] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Neither blocks nor log segments found. Creating new log.
21:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.298884 17400 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.299166 17400 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent starting tablet: real 0.002s	user 0.003s	sys 0.000s
21:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.299206 17402 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Log is configured to *not* fsync() on all Append() calls
21:07:57.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.300839 17402 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: No bootstrap required, opened a new log
21:07:57.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.300930 17402 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent bootstrapping tablet: real 0.005s	user 0.002s	sys 0.000s
21:07:57.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.302606 17402 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.302772 17402 raft_consensus.cc:385] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:57.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.302801 17402 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Initialized, Role: FOLLOWER
21:07:57.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.302901 17402 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.303148 17402 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent starting tablet: real 0.002s	user 0.003s	sys 0.000s
21:07:57.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.340341 17404 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:57.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.340502 17404 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.340915 17404 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.344980 17186 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1" is_pre_election: true
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.345196 17186 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 0.
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.345165 17051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600" is_pre_election: true
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.345280 17051 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 0.
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.345436 17254 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.345613 17404 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.345659 17404 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.345674 17404 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 0 FOLLOWER]: Advancing to term 1
21:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.346267 17404 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.346432 17404 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 1 election: Requested vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.346757 17186 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1"
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.346762 17051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.346843 17051 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 0 FOLLOWER]: Advancing to term 1
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.346841 17186 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 0 FOLLOWER]: Advancing to term 1
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.347633 17186 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 1.
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.347633 17051 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 1.
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.347813 17254 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.347915 17404 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 1 FOLLOWER]: Leader election won for term 1
21:07:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.348089 17404 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 1 LEADER]: Becoming Leader. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Running, Role: LEADER
21:07:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.348176 17404 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.349557 16799 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67). New cstate: current_term: 1 leader_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: UNKNOWN } } }
21:07:57.372 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table dec758c7663d415a98443ac02d75fd83
21:07:57.389 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:07:57.396 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299992, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:07:57.398 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [7ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=null) after lookup
21:07:57.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:57.406693 17235 tablet.cc:2406] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:07:57.410 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:07:57.413 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] Successfully connected to peer
21:07:57.489 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xc0629d45, L:/127.0.0.1:38874 - R:/127.16.75.67:41157] using TOKEN/null
21:07:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.492372 17186 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Refusing update from remote peer 6c0e3990f3964f1e9b57977ffa9d6748: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:07:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.492372 17051 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Refusing update from remote peer 6c0e3990f3964f1e9b57977ffa9d6748: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:07:57.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.492724 17404 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Connected to new peer: Peer: permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:07:57.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.494477 17404 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:07:57.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.498077 17417 mvcc.cc:204] Tried to move back new op lower bound from 7305399818205605888 to 7305399817618694144. Current Snapshot: MvccSnapshot[applied={T|T < 7305399818205605888}]
21:07:57.576 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:07:57.577 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.67:41157
21:07:57.579 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:07:57.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:57.579614 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 17239
21:07:57.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:07:57.589 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] handling channelInactive
21:07:57.589 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state READY due to: connection closed
21:07:57.590 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: connection closed
21:07:57.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:07:57.598 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:07:57.598 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test)) after lookup
21:07:57.599 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:07:57.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.602 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.602 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.617 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:07:57.617 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:07:57.618 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [16ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [16ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376))) after lookup
21:07:57.618 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:07:57.620 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.621 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.637 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:07:57.638 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:07:57.638 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [16ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [16ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [35ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [35ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902))) after lookup
21:07:57.639 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:07:57.641 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.642 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.642 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.697 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:07:57.698 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:07:57.698 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=111), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [16ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [16ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [35ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [35ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [56ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [111ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008))) after lookup
21:07:57.699 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:07:57.701 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.701 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:57.701 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:58.218 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:07:58.218 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:07:58.219 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=631), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [16ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [16ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [35ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [35ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [56ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [111ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [113ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [115ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [115ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [628ms] refreshing cache from master, [629ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [631ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125))) after lookup
21:07:58.220 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:07:58.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:58.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:58.221 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:58.557 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:07:58.557 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:07:58.558 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=971), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [16ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [16ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [35ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [35ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [56ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [111ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [113ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [115ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [115ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [628ms] refreshing cache from master, [629ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [631ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [633ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [635ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [635ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [968ms] refreshing cache from master, [968ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [970ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125) -> (continuation of Deferred@1394769691 after retry RPC@2058597148), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125) -> (continuation of Deferred@1394769691 after retry RPC@2058597148))) after lookup
21:07:58.559 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:07:58.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:58.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:58.560 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:07:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.028086 17429 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:07:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.028223 17429 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.028278 17430 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:07:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.028352 17430 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.028570 17429 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:07:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.028900 17430 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:07:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:59.029050 16985 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:07:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:59.029225 17120 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:07:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:59.029711 17120 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:07:59.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:59.030000 16985 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:07:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032351 17051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600" is_pre_election: true
21:07:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032351 17186 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1" is_pre_election: true
21:07:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032462 17186 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 62a04151dfcd4eaf9bc55e4985fac600 in term 1.
21:07:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032462 17051 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 1.
21:07:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032680 16984 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:07:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032702 17122 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:07:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032829 17429 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:07:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032862 17430 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:07:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032887 17429 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:07:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032898 17430 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:07:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032913 17429 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 1 FOLLOWER]: Advancing to term 2
21:07:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.032922 17430 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 1 FOLLOWER]: Advancing to term 2
21:07:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.033758 17430 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.033775 17429 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.033919 17429 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 2 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.033977 17430 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 2 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034160 17051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034274 17051 raft_consensus.cc:2393] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate c44178f4b5464e1aa2934204708c52a1 in current term 2: Already voted for candidate 62a04151dfcd4eaf9bc55e4985fac600 in this term.
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034301 17186 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1"
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034365 17186 raft_consensus.cc:2393] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 62a04151dfcd4eaf9bc55e4985fac600 in current term 2: Already voted for candidate c44178f4b5464e1aa2934204708c52a1 in this term.
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:59.034461 17120 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034508 17122 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034611 17429 raft_consensus.cc:2749] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:07:59.034821 16985 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:07:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034897 16985 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600; no voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1
21:07:59.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:07:59.034961 17430 raft_consensus.cc:2749] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:08:00.217 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:00.219 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:00.220 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2632), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [16ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [16ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [35ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [35ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [56ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [111ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [113ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [115ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [115ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [628ms] refreshing cache from master, [629ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [631ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [633ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [635ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [635ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [968ms] refreshing cache from master, [968ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [970ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [972ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [974ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [974ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2628ms] refreshing cache from master, [2628ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [2631ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125) -> (continuation of Deferred@1394769691 after retry RPC@2058597148) -> (continuation of Deferred@1315167251 after retry RPC@81337834), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125) -> (continuation of Deferred@1394769691 after retry RPC@2058597148) -> (continuation of Deferred@1315167251 after retry RPC@81337834))) after lookup
21:08:00.220 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:00.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:00.222 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:00.222 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.968075 17439 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:08:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.968201 17439 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.968415 17439 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.968734 17051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600" is_pre_election: true
21:08:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.968845 17051 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 2.
21:08:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.969061 17122 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 
21:08:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.969180 17439 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:08:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.969237 17439 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:08:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:00.969257 17120 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.969259 17439 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 2 FOLLOWER]: Advancing to term 3
21:08:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.969910 17439 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.970052 17439 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 3 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.970227 17051 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:08:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.970309 17051 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 2 FOLLOWER]: Advancing to term 3
21:08:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:00.970572 17120 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.971117 17051 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 3.
21:08:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.971314 17122 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:08:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.971411 17439 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 3 FOLLOWER]: Leader election won for term 3
21:08:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.971503 17439 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 3 LEADER]: Becoming Leader. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Running, Role: LEADER
21:08:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.971598 17439 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:00.972506 16798 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 reported cstate change: term changed from 1 to 3, leader changed from 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67) to c44178f4b5464e1aa2934204708c52a1 (127.16.75.66). New cstate: current_term: 3 leader_uuid: "c44178f4b5464e1aa2934204708c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: HEALTHY } } }
21:08:01.158 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:01.159 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:01.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=3572), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [4ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [16ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [16ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [35ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [35ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [51ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [56ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [108ms] refreshing cache from master, [108ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [111ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [113ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [115ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [115ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [628ms] refreshing cache from master, [629ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [631ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [633ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [635ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [635ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [968ms] refreshing cache from master, [968ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [970ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [972ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [974ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [974ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2628ms] refreshing cache from master, [2628ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [2631ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [2634ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [2636ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2636ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [3568ms] refreshing cache from master, [3569ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [3572ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1830499799(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@7bbe6326 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125) -> (continuation of Deferred@1394769691 after retry RPC@2058597148) -> (continuation of Deferred@1315167251 after retry RPC@81337834) -> (continuation of Deferred@1353226939 after retry RPC@475200265), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@265956eb -> wakeup thread Time-limited test -> (continuation of Deferred@1227846837 after retry RPC@1258141376) -> (continuation of Deferred@687858898 after retry RPC@514830902) -> (continuation of Deferred@1522152152 after retry RPC@1575823008) -> (continuation of Deferred@1074390555 after retry RPC@1770522125) -> (continuation of Deferred@1394769691 after retry RPC@2058597148) -> (continuation of Deferred@1315167251 after retry RPC@81337834) -> (continuation of Deferred@1353226939 after retry RPC@475200265))) after lookup
21:08:01.161 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:01.162 [DEBUG - kudu-nio-3] (Connection.java:757) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] Successfully connected to peer
21:08:01.180 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x7eb32643, L:/127.0.0.1:40290 - R:/127.16.75.66:44849] using TOKEN/null
21:08:01.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.182566 17051 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 3 FOLLOWER]: Refusing update from remote peer c44178f4b5464e1aa2934204708c52a1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
21:08:01.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.182879 17120 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 -> Peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Couldn't send request to peer 6c0e3990f3964f1e9b57977ffa9d6748. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:01.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.182981 17439 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:08:01.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.186429 17458 mvcc.cc:204] Tried to move back new op lower bound from 7305399833321152512 to 7305399832460386304. Current Snapshot: MvccSnapshot[applied={T|T < 7305399833321152512}]
21:08:01.201 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.67:41157
21:08:01.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:08:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.204496 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:01.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.284999 17463 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:01.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.285239 17463 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:01.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.285262 17463 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:01.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.286921 17463 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:01.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.286993 17463 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.67
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.288744 17463 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17463
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.289052 17463 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.289311 17463 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.292017 17468 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.292026 17471 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:01.292035 17469 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:01.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.293048 17463 server_base.cc:1061] running on GCE node
21:08:01.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.293226 17463 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:01.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.293421 17463 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:01.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.294584 17463 hybrid_clock.cc:648] HybridClock initialized: now 1783544881294552 us; error 45 us; skew 500 ppm
21:08:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.295811 17463 webserver.cc:534] Webserver started at http://127.16.75.67:39433/ using document root <none> and password file <none>
21:08:01.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.296025 17463 fs_manager.cc:362] Metadata directory not provided
21:08:01.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.296080 17463 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.297297 17463 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.001s
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.297915 17477 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.298153 17463 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.298233 17463 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.298565 17463 fs_report.cc:389] FS layout report
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/data
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.316200 17463 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.316581 17463 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.316717 17463 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:01.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.316982 17463 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:01.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.317576 17484 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.318696 17463 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.318774 17463 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.318811 17463 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.319509 17463 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.319569 17463 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.319641 17484 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap starting.
21:08:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.327137 17463 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.67:41157
21:08:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.327241 17591 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.67:41157 every 8 connection(s)
21:08:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.327577 17463 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.329708 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 17463
21:08:01.330 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.329854 17484 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Log is configured to *not* fsync() on all Append() calls
21:08:01.335 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.335359 17484 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:01.336 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.66:44849
21:08:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.335711 17484 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap complete.
21:08:01.336 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:08:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.336995 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 17104
21:08:01.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.340106 17484 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent bootstrapping tablet: real 0.021s	user 0.003s	sys 0.011s
21:08:01.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.340631 17592 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:01.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.340728 17592 heartbeater.cc:461] Registering TS with master...
21:08:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.340945 17592 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.340531 17594 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.341378 17594 heartbeater.cc:461] Registering TS with master...
21:08:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.341478 17594 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.341542 16872 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.341919 16798 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.342442 16798 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.67:52233
21:08:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.342728 17593 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.342789 17593 heartbeater.cc:461] Registering TS with master...
21:08:01.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.342887 17593 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:01.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.343246 16726 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.344201 17484 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.344430 17484 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Initialized, Role: FOLLOWER
21:08:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.344574 17484 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.344870 17484 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent starting tablet: real 0.004s	user 0.000s	sys 0.002s
21:08:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:01.345549 17594 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:01.345 [DEBUG - kudu-nio-3] (Connection.java:239) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] handling channelInactive
21:08:01.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:01.346 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state READY due to: connection closed
21:08:01.347 [DEBUG - Time-limited test] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:01.348 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.348 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.348 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.358 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:01.358 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:01.358 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1927952393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@79eb1fee -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b988479 -> wakeup thread Time-limited test)) after lookup
21:08:01.359 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:01.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.361 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:01.377 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:01.378 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [13ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [15ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [15ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1927952393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@79eb1fee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b988479 -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781))) after lookup
21:08:01.379 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:01.380 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.380 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.380 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.417 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:01.417 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:01.418 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [13ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [15ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [15ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [34ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [34ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1927952393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@79eb1fee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b988479 -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461))) after lookup
21:08:01.418 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:01.420 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.420 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.420 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.497 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:01.498 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:01.498 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=152), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [13ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [15ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [15ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [34ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [34ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [72ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [74ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [74ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [151ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1927952393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@79eb1fee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461) -> (continuation of Deferred@340027019 after retry RPC@1313621440), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b988479 -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461) -> (continuation of Deferred@340027019 after retry RPC@1313621440))) after lookup
21:08:01.499 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:01.500 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.500 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.500 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.857 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:01.857 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:01.858 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=511), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [13ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [15ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [15ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [34ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [34ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [72ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [74ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [74ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [151ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [153ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [154ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [155ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [508ms] refreshing cache from master, [508ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [510ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1927952393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@79eb1fee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461) -> (continuation of Deferred@340027019 after retry RPC@1313621440) -> (continuation of Deferred@11093937 after retry RPC@744972321), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b988479 -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461) -> (continuation of Deferred@340027019 after retry RPC@1313621440) -> (continuation of Deferred@11093937 after retry RPC@744972321))) after lookup
21:08:01.858 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:01.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:01.860 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.700675 17606 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader c44178f4b5464e1aa2934204708c52a1)
21:08:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.700780 17606 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.700970 17606 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:02.701707 16984 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111)
21:08:02.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.704525 17546 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" is_pre_election: true
21:08:02.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.704669 17546 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 62a04151dfcd4eaf9bc55e4985fac600 in term 1.
21:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.704905 16985 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748; no voters: c44178f4b5464e1aa2934204708c52a1
21:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.705046 17606 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 3 FOLLOWER]: Leader pre-election won for term 4
21:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.705092 17606 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 3 FOLLOWER]: Starting leader election (detected failure of leader c44178f4b5464e1aa2934204708c52a1)
21:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.705108 17606 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 3 FOLLOWER]: Advancing to term 4
21:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.705662 17606 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.705832 17606 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 4 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.706004 17546 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.706091 17546 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 1 FOLLOWER]: Advancing to term 4
21:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:02.706435 16984 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111)
21:08:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.707203 17546 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 62a04151dfcd4eaf9bc55e4985fac600 in term 4.
21:08:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.707441 16985 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748; no voters: c44178f4b5464e1aa2934204708c52a1
21:08:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.707569 17606 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Leader election won for term 4
21:08:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.707720 17606 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 LEADER]: Becoming Leader. State: Replica: 62a04151dfcd4eaf9bc55e4985fac600, State: Running, Role: LEADER
21:08:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.707862 17606 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:02.708670 16798 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 reported cstate change: term changed from 3 to 4, leader changed from c44178f4b5464e1aa2934204708c52a1 (127.16.75.66) to 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65). New cstate: current_term: 4 leader_uuid: "62a04151dfcd4eaf9bc55e4985fac600" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: UNKNOWN } } }
21:08:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.151612 16984 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 -> Peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Couldn't send request to peer c44178f4b5464e1aa2934204708c52a1. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.253700 17546 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 4 FOLLOWER]: Refusing update from remote peer 62a04151dfcd4eaf9bc55e4985fac600: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
21:08:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.254106 17606 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
21:08:03.317 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:03.318 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:03.318 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1972), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [13ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [15ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [15ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [28ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [31ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [32ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [34ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [34ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [71ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [72ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [74ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [74ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [149ms] refreshing cache from master, [149ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [151ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [153ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [154ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [155ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [508ms] refreshing cache from master, [508ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [510ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [512ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [514ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [514ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [1969ms] refreshing cache from master, [1969ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1971ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1927952393(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@79eb1fee -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461) -> (continuation of Deferred@340027019 after retry RPC@1313621440) -> (continuation of Deferred@11093937 after retry RPC@744972321) -> (continuation of Deferred@934123326 after retry RPC@228365278), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b988479 -> wakeup thread Time-limited test -> (continuation of Deferred@1459647800 after retry RPC@953452781) -> (continuation of Deferred@89077593 after retry RPC@423646461) -> (continuation of Deferred@340027019 after retry RPC@1313621440) -> (continuation of Deferred@11093937 after retry RPC@744972321) -> (continuation of Deferred@934123326 after retry RPC@228365278))) after lookup
21:08:03.319 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:03.320 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] Successfully connected to peer
21:08:03.342 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x1012c7f2, L:/127.0.0.1:35812 - R:/127.16.75.65:37515] using TOKEN/null
21:08:03.354 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.66:44849
21:08:03.354 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:08:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.355407 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:08:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:44849
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:03.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.436853 17634 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:03.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.437036 17634 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:03.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.437057 17634 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:03.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.438478 17634 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:03.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.438534 17634 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.66
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.440110 17634 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:44849
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17634
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.440316 17634 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.440498 17634 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.443200 17639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.443480 17634 server_base.cc:1061] running on GCE node
21:08:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.443238 17640 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:03.443409 17642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.443997 17634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.444236 17634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.445398 17634 hybrid_clock.cc:648] HybridClock initialized: now 1783544883445383 us; error 27 us; skew 500 ppm
21:08:03.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.446587 17634 webserver.cc:534] Webserver started at http://127.16.75.66:39729/ using document root <none> and password file <none>
21:08:03.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.446827 17634 fs_manager.cc:362] Metadata directory not provided
21:08:03.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.446904 17634 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:03.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.448177 17634 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.001s
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.448832 17648 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.449036 17634 fs_manager.cc:730] Time spent opening block manager: real 0.000s	user 0.000s	sys 0.001s
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.449106 17634 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c44178f4b5464e1aa2934204708c52a1"
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.449412 17634 fs_report.cc:389] FS layout report
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/data
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:03.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.462033 17634 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:03.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.462409 17634 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:03.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.462555 17634 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:03.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.462846 17634 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:03.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.463423 17655 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:03.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.464614 17634 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:03.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.464717 17634 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:03.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.464766 17634 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:03.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.465476 17634 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:03.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.465548 17634 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:03.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.465574 17655 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap starting.
21:08:03.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.473802 17634 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.66:44849
21:08:03.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.473857 17762 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.66:44849 every 8 connection(s)
21:08:03.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.474259 17634 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:03.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.476336 17655 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Log is configured to *not* fsync() on all Append() calls
21:08:03.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.479707 17655 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:03.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.480221 17655 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap complete.
21:08:03.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.480753 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 17634
21:08:03.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:03.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.481485 17655 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent bootstrapping tablet: real 0.016s	user 0.008s	sys 0.006s
21:08:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.482795 17763 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.482916 17763 heartbeater.cc:461] Registering TS with master...
21:08:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.483160 17763 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.483129 17655 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.483403 17655 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Initialized, Role: FOLLOWER
21:08:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.483522 17655 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:03.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.484030 17655 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.002s
21:08:03.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.484232 17765 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:03.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.484288 17765 heartbeater.cc:461] Registering TS with master...
21:08:03.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.484402 17765 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:03.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.484592 16871 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:03.486 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.486105 16798 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.486634 16798 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.66:57425
21:08:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.487411 17765 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:03.487 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.65:37515
21:08:03.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:08:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.488245 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 16969
21:08:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.489688 17764 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.489776 17764 heartbeater.cc:461] Registering TS with master...
21:08:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.489850 17764 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:03.490326 16727 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:03.495 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:03.495 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] handling channelInactive
21:08:03.495 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state READY due to: connection closed
21:08:03.496 [DEBUG - Time-limited test] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:03.497 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.498 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.498 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.517 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:03.518 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:03.518 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [21ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1226886646(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@20cba56f -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5212960 -> wakeup thread Time-limited test)) after lookup
21:08:03.519 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:03.520 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.520 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.520 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.537 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:03.537 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:03.538 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [21ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [22ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [24ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [24ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1226886646(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@20cba56f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5212960 -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089))) after lookup
21:08:03.538 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:03.539 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.539 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.539 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.557 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:03.557 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:03.558 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [21ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [22ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [24ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [24ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [42ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1226886646(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@20cba56f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5212960 -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937))) after lookup
21:08:03.558 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:03.559 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.559 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.559 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.677 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:03.678 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:03.678 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=182), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [21ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [22ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [24ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [24ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [42ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [63ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [63ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [181ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1226886646(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@20cba56f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5212960 -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647))) after lookup
21:08:03.679 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:03.680 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.680 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.680 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.957 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:03.957 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:03.958 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=462), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [21ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [22ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [24ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [24ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [42ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [63ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [63ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [181ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [182ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [184ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [184ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [458ms] refreshing cache from master, [458ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [461ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1226886646(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@20cba56f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647) -> (continuation of Deferred@213081805 after retry RPC@668074464), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5212960 -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647) -> (continuation of Deferred@213081805 after retry RPC@668074464))) after lookup
21:08:03.958 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:03.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:03.960 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:04.677 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:04.678 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:04.678 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1182), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [21ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [22ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [24ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [24ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [42ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [63ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [63ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [181ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [182ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [184ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [184ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [458ms] refreshing cache from master, [458ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [461ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [462ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [464ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [464ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1179ms] refreshing cache from master, [1179ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1181ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1226886646(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@20cba56f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647) -> (continuation of Deferred@213081805 after retry RPC@668074464) -> (continuation of Deferred@871370206 after retry RPC@68863809), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5212960 -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647) -> (continuation of Deferred@213081805 after retry RPC@668074464) -> (continuation of Deferred@871370206 after retry RPC@68863809))) after lookup
21:08:04.679 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:04.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:04.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:04.680 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.853515 17776 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 62a04151dfcd4eaf9bc55e4985fac600)
21:08:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.853622 17776 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.854018 17776 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:04.854470 17481 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:04.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:04.854960 17481 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:04.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.857544 17717 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1" is_pre_election: true
21:08:04.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.857681 17717 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 3.
21:08:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.857904 17478 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600
21:08:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.858031 17776 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 4 FOLLOWER]: Leader pre-election won for term 5
21:08:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.858090 17776 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 62a04151dfcd4eaf9bc55e4985fac600)
21:08:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.858107 17776 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 4 FOLLOWER]: Advancing to term 5
21:08:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.858682 17776 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.858837 17776 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 5 election: Requested vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.859014 17717 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1"
21:08:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.859119 17717 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 3 FOLLOWER]: Advancing to term 5
21:08:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:04.859599 17481 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:04.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.860404 17717 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 5.
21:08:04.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.860612 17478 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600
21:08:04.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.860715 17776 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 5 FOLLOWER]: Leader election won for term 5
21:08:04.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.860850 17776 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 5 LEADER]: Becoming Leader. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Running, Role: LEADER
21:08:04.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.860917 17776 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:04.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:04.861692 16798 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 reported cstate change: term changed from 4 to 5, leader changed from 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65) to 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67). New cstate: current_term: 5 leader_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: UNKNOWN } } }
21:08:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:05.278786 17717 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 5 FOLLOWER]: Refusing update from remote peer 6c0e3990f3964f1e9b57977ffa9d6748: Log matching property violated. Preceding OpId in replica: term: 3 index: 8. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
21:08:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:05.279217 17776 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
21:08:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:05.282465 17481 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 -> Peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Couldn't send request to peer 62a04151dfcd4eaf9bc55e4985fac600. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:06.658 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:06.658 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:06.659 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=3163), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [21ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [22ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [24ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [24ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [42ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [63ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [63ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [179ms] refreshing cache from master, [179ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [181ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [182ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [184ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [184ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [458ms] refreshing cache from master, [458ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [461ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [462ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [464ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [464ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1179ms] refreshing cache from master, [1179ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1181ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [1182ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1184ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1184ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [3158ms] refreshing cache from master, [3159ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [3161ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1226886646(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@20cba56f -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647) -> (continuation of Deferred@213081805 after retry RPC@668074464) -> (continuation of Deferred@871370206 after retry RPC@68863809) -> (continuation of Deferred@1457461560 after retry RPC@1704969639), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5212960 -> wakeup thread Time-limited test -> (continuation of Deferred@1490754387 after retry RPC@1413305089) -> (continuation of Deferred@445386922 after retry RPC@605547937) -> (continuation of Deferred@1330543122 after retry RPC@1420819647) -> (continuation of Deferred@213081805 after retry RPC@668074464) -> (continuation of Deferred@871370206 after retry RPC@68863809) -> (continuation of Deferred@1457461560 after retry RPC@1704969639))) after lookup
21:08:06.660 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:06.662 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] Successfully connected to peer
21:08:06.677 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x7cdbb3e2, L:/127.0.0.1:44814 - R:/127.16.75.67:41157] using TOKEN/null
21:08:06.692 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.65:37515
21:08:06.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.693015 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:37515
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46433
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.772703 17815 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.772926 17815 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.772961 17815 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.774750 17815 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.774878 17815 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.65
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.776782 17815 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:37515
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46433
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17815
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:08:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.777130 17815 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.777441 17815 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.780718 17820 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.780733 17823 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:06.780761 17821 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.781086 17815 server_base.cc:1061] running on GCE node
21:08:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.781282 17815 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.781563 17815 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:06.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.782723 17815 hybrid_clock.cc:648] HybridClock initialized: now 1783544886782694 us; error 43 us; skew 500 ppm
21:08:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.784046 17815 webserver.cc:534] Webserver started at http://127.16.75.65:46433/ using document root <none> and password file <none>
21:08:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.784287 17815 fs_manager.cc:362] Metadata directory not provided
21:08:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.784392 17815 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.785691 17815 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
21:08:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.786342 17829 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.786576 17815 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.786669 17815 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.786981 17815 fs_report.cc:389] FS layout report
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/data
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.798982 17815 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.799391 17815 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.799583 17815 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.799927 17815 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.800482 17836 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.801488 17815 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.801538 17815 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.801566 17815 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:06.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.802081 17815 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:06.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.802109 17815 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:06.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.802259 17836 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap starting.
21:08:06.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.809796 17815 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.65:37515
21:08:06.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.809883 17943 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.65:37515 every 8 connection(s)
21:08:06.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.810182 17815 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.812216 17836 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Log is configured to *not* fsync() on all Append() calls
21:08:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.815168 17836 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.815486 17836 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap complete.
21:08:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.816447 17836 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent bootstrapping tablet: real 0.014s	user 0.008s	sys 0.006s
21:08:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.817260 17836 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.817441 17836 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62a04151dfcd4eaf9bc55e4985fac600, State: Initialized, Role: FOLLOWER
21:08:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.817546 17836 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.817790 17836 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent starting tablet: real 0.001s	user 0.000s	sys 0.002s
21:08:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.818472 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 17815
21:08:06.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.818749 17945 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.818856 17945 heartbeater.cc:461] Registering TS with master...
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.818849 17944 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.818944 17944 heartbeater.cc:461] Registering TS with master...
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.819049 17944 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.819053 17945 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.819654 16727 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.819703 16871 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.820310 17946 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.820379 17946 heartbeater.cc:461] Registering TS with master...
21:08:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.820477 17946 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.820758 16798 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:06.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.821234 16798 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.65:54751
21:08:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.821888 17946 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:06.823 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:06.829 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.67:41157
21:08:06.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:08:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:06.829843 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 17463
21:08:06.836 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:06.836 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] handling channelInactive
21:08:06.836 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state READY due to: connection closed
21:08:06.856 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:06.857 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.858 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.858 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.878 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:06.878 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:06.878 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [22ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@246283056(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1504a5bf -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12649c9e -> wakeup thread Time-limited test)) after lookup
21:08:06.879 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:06.881 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.881 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.881 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.897 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:06.897 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:06.898 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [22ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [43ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [45ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@246283056(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1504a5bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12649c9e -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454))) after lookup
21:08:06.899 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:06.900 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.900 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.900 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.918 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:06.918 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:06.918 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [22ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [43ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [45ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [64ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [81ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@246283056(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1504a5bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12649c9e -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263))) after lookup
21:08:06.919 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:06.921 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.921 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:06.921 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:07.197 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:07.198 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:07.198 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=361), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [22ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [43ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [45ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [64ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [81ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [83ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [85ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [85ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [359ms] refreshing cache from master, [359ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [361ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@246283056(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1504a5bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263) -> (continuation of Deferred@1339194725 after retry RPC@1679893487), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12649c9e -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263) -> (continuation of Deferred@1339194725 after retry RPC@1679893487))) after lookup
21:08:07.199 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:07.200 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:07.200 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:07.200 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.010665 17957 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:08:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.010864 17957 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.011231 17957 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:08.011792 17831 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:08.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:08.012593 17831 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.014602 17717 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1" is_pre_election: true
21:08:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.014964 17830 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600; no voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1
21:08:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.015137 17957 raft_consensus.cc:2749] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:08:08.097 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:08.097 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:08.098 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1261), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [22ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [43ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [45ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [64ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [81ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [83ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [85ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [85ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [359ms] refreshing cache from master, [359ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [361ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [362ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [364ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [364ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [1258ms] refreshing cache from master, [1259ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1261ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@246283056(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1504a5bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263) -> (continuation of Deferred@1339194725 after retry RPC@1679893487) -> (continuation of Deferred@1321243703 after retry RPC@588507227), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12649c9e -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263) -> (continuation of Deferred@1339194725 after retry RPC@1679893487) -> (continuation of Deferred@1321243703 after retry RPC@588507227))) after lookup
21:08:08.098 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:08.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:08.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:08.100 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:08.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.191416 17962 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:08:08.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.191545 17962 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.191913 17962 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:08.192394 17650 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:08.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:08.192873 17650 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.195055 17898 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600" is_pre_election: true
21:08:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.195186 17898 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 4.
21:08:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.195488 17652 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:08:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.195658 17962 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 5 FOLLOWER]: Leader pre-election won for term 6
21:08:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.195716 17962 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:08:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.195731 17962 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 5 FOLLOWER]: Advancing to term 6
21:08:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.196411 17962 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.196574 17962 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 6 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.196772 17898 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:08:08.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.196877 17898 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 4 FOLLOWER]: Advancing to term 6
21:08:08.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:08.197242 17650 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.197966 17898 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 6.
21:08:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.198154 17652 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:08:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.198279 17962 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Leader election won for term 6
21:08:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.198483 17962 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 LEADER]: Becoming Leader. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Running, Role: LEADER
21:08:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.198589 17962 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.199368 16798 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 reported cstate change: term changed from 5 to 6, leader changed from 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67) to c44178f4b5464e1aa2934204708c52a1 (127.16.75.66). New cstate: current_term: 6 leader_uuid: "c44178f4b5464e1aa2934204708c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: HEALTHY } } }
21:08:08.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.615309 17898 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 6 FOLLOWER]: Refusing update from remote peer c44178f4b5464e1aa2934204708c52a1: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 17. (index mismatch)
21:08:08.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:08.615808 17962 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
21:08:08.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:08.619526 17650 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 -> Peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Couldn't send request to peer 6c0e3990f3964f1e9b57977ffa9d6748. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:09.397 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:09.397 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:09.397 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2561), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [22ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [41ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [43ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [45ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [45ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [58ms] refreshing cache from master, [58ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [61ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [62ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [64ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [64ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [81ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [83ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [85ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [85ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [359ms] refreshing cache from master, [359ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [361ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [362ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [364ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [364ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [1258ms] refreshing cache from master, [1259ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1261ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [1262ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1264ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [1264ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2558ms] refreshing cache from master, [2558ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [2560ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@246283056(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1504a5bf -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263) -> (continuation of Deferred@1339194725 after retry RPC@1679893487) -> (continuation of Deferred@1321243703 after retry RPC@588507227) -> (continuation of Deferred@571167898 after retry RPC@518398384), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12649c9e -> wakeup thread Time-limited test -> (continuation of Deferred@1249038898 after retry RPC@2084845454) -> (continuation of Deferred@1774601614 after retry RPC@887355263) -> (continuation of Deferred@1339194725 after retry RPC@1679893487) -> (continuation of Deferred@1321243703 after retry RPC@588507227) -> (continuation of Deferred@571167898 after retry RPC@518398384))) after lookup
21:08:09.398 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:09.399 [DEBUG - kudu-nio-1] (Connection.java:757) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] Successfully connected to peer
21:08:09.416 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xcee2427d, L:/127.0.0.1:39594 - R:/127.16.75.66:44849] using TOKEN/null
21:08:09.427 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.67:41157
21:08:09.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.428153 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.508244 17993 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.508432 17993 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.508453 17993 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.509876 17993 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.509927 17993 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.67
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.511368 17993 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.17993
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.511572 17993 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.511759 17993 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.514549 17999 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.514554 17998 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:09.514621 18001 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.515117 17993 server_base.cc:1061] running on GCE node
21:08:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.515285 17993 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.515532 17993 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:09.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.516688 17993 hybrid_clock.cc:648] HybridClock initialized: now 1783544889516665 us; error 43 us; skew 500 ppm
21:08:09.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.517879 17993 webserver.cc:534] Webserver started at http://127.16.75.67:39433/ using document root <none> and password file <none>
21:08:09.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.518115 17993 fs_manager.cc:362] Metadata directory not provided
21:08:09.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.518203 17993 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:09.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.519395 17993 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
21:08:09.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.520402 18007 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:09.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.520668 17993 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.520776 17993 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.521130 17993 fs_report.cc:389] FS layout report
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/data
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.531327 17993 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.531697 17993 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:09.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.531911 17993 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:09.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.532160 17993 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:09.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.532704 18014 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:09.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.533526 17993 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:09.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.533589 17993 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:09.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.533625 17993 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.534193 17993 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.534248 17993 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.534286 18014 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap starting.
21:08:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.541348 17993 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.67:41157
21:08:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.541417 18121 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.67:41157 every 8 connection(s)
21:08:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.541759 17993 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.542932 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 17993
21:08:09.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.544281 18014 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Log is configured to *not* fsync() on all Append() calls
21:08:09.548 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.548995 18124 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.549121 18124 heartbeater.cc:461] Registering TS with master...
21:08:09.549 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.66:44849
21:08:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.549335 18124 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:09.550 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:08:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.549921 16798 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.550217 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 17634
21:08:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.550431 16798 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.67:51947
21:08:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.552369 18122 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.552451 18122 heartbeater.cc:461] Registering TS with master...
21:08:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.552512 18122 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.552977 16871 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.553304 18123 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.553365 18123 heartbeater.cc:461] Registering TS with master...
21:08:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.553400 18123 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.553694 16727 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.555459 18014 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.555725 18014 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap complete.
21:08:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.556886 18014 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent bootstrapping tablet: real 0.023s	user 0.007s	sys 0.007s
21:08:09.557 [DEBUG - kudu-nio-1] (Connection.java:239) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] handling channelInactive
21:08:09.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state READY due to: connection closed
21:08:09.558 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.558214 18014 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.558502 18014 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Initialized, Role: FOLLOWER
21:08:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.558620 18014 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:09.559 [DEBUG - Time-limited test] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.558887 18014 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.002s
21:08:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:09.559273 18124 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:09.560 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.560 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.560 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.577 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:09.577 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:09.577 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@697405867(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@50ce0bbc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2483aad7 -> wakeup thread Time-limited test)) after lookup
21:08:09.578 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:09.579 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.579 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.579 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:09.598 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:09.598 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [19ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [21ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [21ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@697405867(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@50ce0bbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2483aad7 -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666))) after lookup
21:08:09.599 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:09.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.600 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.638 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:09.638 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:09.639 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [19ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [21ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [21ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [42ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [42ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [79ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@697405867(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@50ce0bbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2483aad7 -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964))) after lookup
21:08:09.639 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:09.641 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.641 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.641 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.777 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:09.777 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:09.778 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=219), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [19ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [21ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [21ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [42ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [42ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [79ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [81ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [83ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [83ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [216ms] refreshing cache from master, [217ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [219ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@697405867(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@50ce0bbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964) -> (continuation of Deferred@1151338718 after retry RPC@2110207245), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2483aad7 -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964) -> (continuation of Deferred@1151338718 after retry RPC@2110207245))) after lookup
21:08:09.778 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:09.780 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.780 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:09.780 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:10.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:10.258 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:10.258 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=700), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [19ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [21ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [21ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [42ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [42ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [79ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [81ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [83ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [83ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [216ms] refreshing cache from master, [217ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [219ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [220ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [222ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [222ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [697ms] refreshing cache from master, [697ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [699ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@697405867(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@50ce0bbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964) -> (continuation of Deferred@1151338718 after retry RPC@2110207245) -> (continuation of Deferred@1031989696 after retry RPC@2032782368), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2483aad7 -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964) -> (continuation of Deferred@1151338718 after retry RPC@2110207245) -> (continuation of Deferred@1031989696 after retry RPC@2032782368))) after lookup
21:08:10.259 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:10.260 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:10.260 [DEBUG - kudu-nio-7] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:10.261 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.927150 18135 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader c44178f4b5464e1aa2934204708c52a1)
21:08:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.927268 18135 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.927469 18135 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:10.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:10.928498 17830 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111)
21:08:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.930907 18076 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" is_pre_election: true
21:08:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.931078 18076 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 62a04151dfcd4eaf9bc55e4985fac600 in term 5.
21:08:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.931370 17831 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748; no voters: c44178f4b5464e1aa2934204708c52a1
21:08:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.931485 18135 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 6 FOLLOWER]: Leader pre-election won for term 7
21:08:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.931529 18135 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 6 FOLLOWER]: Starting leader election (detected failure of leader c44178f4b5464e1aa2934204708c52a1)
21:08:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.931545 18135 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 6 FOLLOWER]: Advancing to term 7
21:08:10.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.932186 18135 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:10.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.932338 18135 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 7 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:10.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.932462 18076 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:08:10.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.932561 18076 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 5 FOLLOWER]: Advancing to term 7
21:08:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:10.932951 17830 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111)
21:08:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.933673 18076 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 62a04151dfcd4eaf9bc55e4985fac600 in term 7.
21:08:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.933847 17831 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748; no voters: c44178f4b5464e1aa2934204708c52a1
21:08:10.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.933959 18135 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 7 FOLLOWER]: Leader election won for term 7
21:08:10.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.934087 18135 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 7 LEADER]: Becoming Leader. State: Replica: 62a04151dfcd4eaf9bc55e4985fac600, State: Running, Role: LEADER
21:08:10.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.934149 18135 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:10.934969 16798 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 reported cstate change: term changed from 6 to 7, leader changed from c44178f4b5464e1aa2934204708c52a1 (127.16.75.66) to 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65). New cstate: current_term: 7 leader_uuid: "62a04151dfcd4eaf9bc55e4985fac600" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: UNKNOWN } } }
21:08:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.473659 18076 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 7 FOLLOWER]: Refusing update from remote peer 62a04151dfcd4eaf9bc55e4985fac600: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
21:08:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.474084 18152 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
21:08:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.477243 17830 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 -> Peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Couldn't send request to peer c44178f4b5464e1aa2934204708c52a1. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:11.677 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:11.678 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:11.679 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2120), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [2ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [18ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [19ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [21ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [21ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [42ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [42ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [79ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [81ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [83ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [83ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [216ms] refreshing cache from master, [217ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [219ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [220ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [222ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [222ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [697ms] refreshing cache from master, [697ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [699ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [700ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [703ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [703ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [2116ms] refreshing cache from master, [2116ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [2119ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@697405867(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@50ce0bbc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964) -> (continuation of Deferred@1151338718 after retry RPC@2110207245) -> (continuation of Deferred@1031989696 after retry RPC@2032782368) -> (continuation of Deferred@1097325321 after retry RPC@1981302729), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2483aad7 -> wakeup thread Time-limited test -> (continuation of Deferred@673064140 after retry RPC@292742666) -> (continuation of Deferred@600271720 after retry RPC@629377964) -> (continuation of Deferred@1151338718 after retry RPC@2110207245) -> (continuation of Deferred@1031989696 after retry RPC@2032782368) -> (continuation of Deferred@1097325321 after retry RPC@1981302729))) after lookup
21:08:11.679 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:11.680 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] Successfully connected to peer
21:08:11.698 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd9d9de93, L:/127.0.0.1:49428 - R:/127.16.75.65:37515] using TOKEN/null
21:08:11.712 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.66:44849
21:08:11.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:08:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.714751 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:44849
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.801586 18167 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.801761 18167 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.801782 18167 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.803226 18167 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.803277 18167 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.66
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.804845 18167 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:44849
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18167
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.805043 18167 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.805218 18167 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.807587 18172 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.807703 18167 server_base.cc:1061] running on GCE node
21:08:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.807657 18173 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:11.807587 18175 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.808197 18167 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.808398 18167 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.809527 18167 hybrid_clock.cc:648] HybridClock initialized: now 1783544891809504 us; error 39 us; skew 500 ppm
21:08:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.810732 18167 webserver.cc:534] Webserver started at http://127.16.75.66:39729/ using document root <none> and password file <none>
21:08:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.810937 18167 fs_manager.cc:362] Metadata directory not provided
21:08:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.810994 18167 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.812438 18167 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.002s	sys 0.000s
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.813032 18181 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.813239 18167 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.813287 18167 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c44178f4b5464e1aa2934204708c52a1"
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.813530 18167 fs_report.cc:389] FS layout report
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/data
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.837704 18167 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.838027 18167 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.838121 18167 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.838318 18167 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.838729 18188 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:11.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.839700 18167 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:11.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.839756 18167 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:11.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.839785 18167 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:11.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.840409 18167 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:11.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.840447 18167 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:11.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.840579 18188 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap starting.
21:08:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.847906 18167 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.66:44849
21:08:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.847963 18295 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.66:44849 every 8 connection(s)
21:08:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.848338 18167 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:11.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.850661 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 18167
21:08:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.850875 18188 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Log is configured to *not* fsync() on all Append() calls
21:08:11.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.855794 18188 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.856230 18188 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap complete.
21:08:11.856 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:11.857 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.65:37515
21:08:11.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.857136 18188 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent bootstrapping tablet: real 0.017s	user 0.000s	sys 0.014s
21:08:11.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:08:11.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.857802 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 17815
21:08:11.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.859058 18188 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:11.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.859390 18188 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Initialized, Role: FOLLOWER
21:08:11.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.859529 18188 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.859794 18188 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent starting tablet: real 0.003s	user 0.000s	sys 0.003s
21:08:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.860239 18297 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.860352 18297 heartbeater.cc:461] Registering TS with master...
21:08:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.860581 18297 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.861303 16727 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.861909 18298 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.862087 18298 heartbeater.cc:461] Registering TS with master...
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.862115 18296 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.862154 18296 heartbeater.cc:461] Registering TS with master...
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.862193 18298 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.862222 18296 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.862476 16799 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.862836 16871 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:11.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.863057 16799 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.66:35287
21:08:11.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:11.863744 18298 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:11.864 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:11.864 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] handling channelInactive
21:08:11.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state READY due to: connection closed
21:08:11.865 [DEBUG - Time-limited test] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:11.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.867 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.876 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:11.877 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:11.877 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@658680155(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1801ad9c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@59547b8f -> wakeup thread Time-limited test)) after lookup
21:08:11.877 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:11.879 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.879 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.879 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.897 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:11.898 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:11.898 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [14ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [14ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@658680155(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1801ad9c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@59547b8f -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941))) after lookup
21:08:11.899 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:11.900 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.900 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.900 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.917 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:11.917 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:11.918 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [14ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [14ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [33ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [35ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [35ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [49ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [52ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@658680155(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1801ad9c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@59547b8f -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820))) after lookup
21:08:11.918 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:11.919 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.920 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.920 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.997 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:11.997 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:11.998 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=132), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [14ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [14ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [33ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [35ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [35ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [49ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [52ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [55ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [55ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@658680155(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1801ad9c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@59547b8f -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502))) after lookup
21:08:11.998 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:11.999 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:11.999 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:12.000 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:12.037 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:12.038 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:12.038 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=173), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [14ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [14ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [33ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [35ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [35ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [49ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [52ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [55ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [55ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [133ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [135ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [135ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [172ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@658680155(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1801ad9c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502) -> (continuation of Deferred@591870831 after retry RPC@482194097), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@59547b8f -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502) -> (continuation of Deferred@591870831 after retry RPC@482194097))) after lookup
21:08:12.038 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:12.039 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:12.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:12.040 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.196108 18312 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:08:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.196246 18312 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.196853 18312 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:13.197 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:13.197 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:13.197067 18185 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:13.198 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1333), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [14ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [14ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [33ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [35ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [35ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [49ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [52ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [55ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [55ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [133ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [135ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [135ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [172ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [173ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [175ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [175ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1329ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1332ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@658680155(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1801ad9c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502) -> (continuation of Deferred@591870831 after retry RPC@482194097) -> (continuation of Deferred@275776202 after retry RPC@1638466127), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@59547b8f -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502) -> (continuation of Deferred@591870831 after retry RPC@482194097) -> (continuation of Deferred@275776202 after retry RPC@1638466127))) after lookup
21:08:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:13.198349 18185 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:13.199 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:13.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.201092 18076 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" is_pre_election: true
21:08:13.201 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:13.201 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:13.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.201509 18183 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748
21:08:13.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.201711 18312 raft_consensus.cc:2749] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
21:08:13.201 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.211592 18317 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 62a04151dfcd4eaf9bc55e4985fac600)
21:08:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.211737 18317 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.212188 18317 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:13.212674 18011 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:13.213205 18011 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:13.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.215600 18250 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1" is_pre_election: true
21:08:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.215764 18250 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 6.
21:08:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.216063 18008 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600
21:08:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.216243 18317 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 7 FOLLOWER]: Leader pre-election won for term 8
21:08:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.216356 18317 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 62a04151dfcd4eaf9bc55e4985fac600)
21:08:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.216382 18317 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 7 FOLLOWER]: Advancing to term 8
21:08:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.217226 18317 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.217401 18317 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 8 election: Requested vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.217653 18250 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1"
21:08:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.217772 18250 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 6 FOLLOWER]: Advancing to term 8
21:08:13.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:13.218286 18011 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.218946 18250 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 8.
21:08:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.219156 18008 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600
21:08:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.219261 18317 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 8 FOLLOWER]: Leader election won for term 8
21:08:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.219434 18317 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 8 LEADER]: Becoming Leader. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Running, Role: LEADER
21:08:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.219527 18317 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.220299 16799 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 reported cstate change: term changed from 7 to 8, leader changed from 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65) to 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67). New cstate: current_term: 8 leader_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: UNKNOWN } } }
21:08:13.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:13.712538 18011 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 -> Peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Couldn't send request to peer 62a04151dfcd4eaf9bc55e4985fac600. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.750355 18250 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 8 FOLLOWER]: Refusing update from remote peer 6c0e3990f3964f1e9b57977ffa9d6748: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
21:08:13.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:13.750800 18317 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
21:08:15.137 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:15.137 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:15.138 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=3272), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [2ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [11ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [12ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [14ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [14ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [32ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [33ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [35ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [35ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [49ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [52ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [53ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [55ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [55ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [132ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [133ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [135ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [135ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [170ms] refreshing cache from master, [170ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [172ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [173ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [175ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [175ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1329ms] refreshing cache from master, [1329ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1332ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [1334ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1336ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1337ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [3269ms] refreshing cache from master, [3269ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [3271ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@658680155(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1801ad9c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502) -> (continuation of Deferred@591870831 after retry RPC@482194097) -> (continuation of Deferred@275776202 after retry RPC@1638466127) -> (continuation of Deferred@270661161 after retry RPC@968041752), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@59547b8f -> wakeup thread Time-limited test -> (continuation of Deferred@445190696 after retry RPC@1627101941) -> (continuation of Deferred@1402889256 after retry RPC@1871601820) -> (continuation of Deferred@907322400 after retry RPC@704821502) -> (continuation of Deferred@591870831 after retry RPC@482194097) -> (continuation of Deferred@275776202 after retry RPC@1638466127) -> (continuation of Deferred@270661161 after retry RPC@968041752))) after lookup
21:08:15.138 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:15.139 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] Successfully connected to peer
21:08:15.157 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4741f055, L:/127.0.0.1:48814 - R:/127.16.75.67:41157] using TOKEN/null
21:08:15.168 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.65:37515
21:08:15.169 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.169375 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:37515
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46433
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.251593 18353 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.251834 18353 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.251943 18353 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.253628 18353 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.253721 18353 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.65
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.255779 18353 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:37515
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46433
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18353
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.256119 18353 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.256325 18353 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.259207 18361 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.259163 18359 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:15.259163 18358 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.259488 18353 server_base.cc:1061] running on GCE node
21:08:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.259708 18353 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:15.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.260049 18353 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:15.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.261241 18353 hybrid_clock.cc:648] HybridClock initialized: now 1783544895261207 us; error 49 us; skew 500 ppm
21:08:15.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.262655 18353 webserver.cc:534] Webserver started at http://127.16.75.65:46433/ using document root <none> and password file <none>
21:08:15.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.262887 18353 fs_manager.cc:362] Metadata directory not provided
21:08:15.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.262964 18353 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.264617 18353 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.001s
21:08:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.265355 18367 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.265533 18353 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.265600 18353 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.265969 18353 fs_report.cc:389] FS layout report
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/data
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:15.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.283550 18353 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.283964 18353 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.284071 18353 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.284262 18353 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.284782 18374 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.285789 18353 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.285847 18353 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.285882 18353 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.286403 18353 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.286437 18353 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.286556 18374 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap starting.
21:08:15.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.294170 18353 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.65:37515
21:08:15.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.294245 18481 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.65:37515 every 8 connection(s)
21:08:15.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.294602 18353 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:15.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.294745 18374 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Log is configured to *not* fsync() on all Append() calls
21:08:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.298837 18374 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.299208 18374 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap complete.
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.300876 18374 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent bootstrapping tablet: real 0.014s	user 0.013s	sys 0.000s
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.300967 18484 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301299 18484 heartbeater.cc:461] Registering TS with master...
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301376 18482 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301422 18483 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301427 18482 heartbeater.cc:461] Registering TS with master...
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301498 18483 heartbeater.cc:461] Registering TS with master...
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301545 18482 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301544 18484 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.301613 18483 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.302625 16798 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.302650 16726 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.302711 16871 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.303186 16798 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.65:39091
21:08:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.303018 18374 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.303278 18374 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62a04151dfcd4eaf9bc55e4985fac600, State: Initialized, Role: FOLLOWER
21:08:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.303395 18374 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.303668 18374 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.000s
21:08:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.303985 18484 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.304801 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 18353
21:08:15.305 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:15.309 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:15.310 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.67:41157
21:08:15.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:08:15.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:15.312002 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 17993
21:08:15.317 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:15.318 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] handling channelInactive
21:08:15.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state READY due to: connection closed
21:08:15.335 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:15.336 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.336 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.336 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.357 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:15.357 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:15.357 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=3, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [18ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [18ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@262306295(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1e0f8876 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@69c668e7 -> wakeup thread Time-limited test)) after lookup
21:08:15.358 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:15.359 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.359 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.359 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:15.377 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:15.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=5, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [18ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [18ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [41ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@262306295(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1e0f8876 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@69c668e7 -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339))) after lookup
21:08:15.378 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:15.379 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.379 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.379 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.416 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:15.417 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:15.417 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=7, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=99), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [18ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [18ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [41ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [61ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [61ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [98ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@262306295(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1e0f8876 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@69c668e7 -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267))) after lookup
21:08:15.418 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:15.419 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.419 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.419 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.497 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:15.497 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:15.497 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=9, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=179), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [18ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [18ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [41ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [61ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [61ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [98ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [99ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [101ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [101ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [178ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@262306295(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1e0f8876 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@69c668e7 -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312))) after lookup
21:08:15.498 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:15.499 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.499 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.500 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.757 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:15.757 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:15.758 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=11, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=439), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [18ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [18ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [41ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [61ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [61ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [98ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [99ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [101ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [101ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [178ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [180ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [181ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [182ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [437ms] refreshing cache from master, [437ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [439ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@262306295(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1e0f8876 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312) -> (continuation of Deferred@970796492 after retry RPC@241792825), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@69c668e7 -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312) -> (continuation of Deferred@970796492 after retry RPC@241792825))) after lookup
21:08:15.759 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:15.760 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.760 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:15.760 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:16.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:16.597 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:16.598 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=13, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1280), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [18ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [18ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [41ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [61ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [61ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [98ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [99ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [101ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [101ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [178ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [180ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [181ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [182ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [437ms] refreshing cache from master, [437ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [439ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [440ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [442ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [442ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [1276ms] refreshing cache from master, [1277ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1279ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@262306295(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1e0f8876 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312) -> (continuation of Deferred@970796492 after retry RPC@241792825) -> (continuation of Deferred@81239517 after retry RPC@1103291073), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@69c668e7 -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312) -> (continuation of Deferred@970796492 after retry RPC@241792825) -> (continuation of Deferred@81239517 after retry RPC@1103291073))) after lookup
21:08:16.599 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:16.600 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:16.600 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:16.600 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.668220 18496 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:08:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.668320 18496 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.668519 18496 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:16.669438 18183 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.672075 18436 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600" is_pre_election: true
21:08:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.672217 18436 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 7.
21:08:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.672452 18185 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:08:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.672550 18496 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 8 FOLLOWER]: Leader pre-election won for term 9
21:08:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.672600 18496 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:08:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.672640 18496 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 8 FOLLOWER]: Advancing to term 9
21:08:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.673231 18496 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.673378 18496 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 9 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.673558 18436 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:08:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.673662 18436 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 7 FOLLOWER]: Advancing to term 9
21:08:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:16.673935 18183 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.674829 18436 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 9.
21:08:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.675072 18185 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:08:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.675184 18496 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 9 FOLLOWER]: Leader election won for term 9
21:08:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.675333 18496 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 9 LEADER]: Becoming Leader. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Running, Role: LEADER
21:08:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.675452 18496 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:16.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:16.676393 16798 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 reported cstate change: term changed from 8 to 9, leader changed from 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67) to c44178f4b5464e1aa2934204708c52a1 (127.16.75.66). New cstate: current_term: 9 leader_uuid: "c44178f4b5464e1aa2934204708c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: HEALTHY } } }
21:08:17.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.157673 18183 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 -> Peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Couldn't send request to peer 6c0e3990f3964f1e9b57977ffa9d6748. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:17.256 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:17.257 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:17.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=15, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1939), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [18ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [18ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [40ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [41ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [41ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [61ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [61ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [98ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [99ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [101ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [101ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [178ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [180ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [181ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [182ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [437ms] refreshing cache from master, [437ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [439ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [440ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [442ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [442ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [1276ms] refreshing cache from master, [1277ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1279ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [1280ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1282ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [1282ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [1936ms] refreshing cache from master, [1936ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [1938ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@262306295(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@1e0f8876 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312) -> (continuation of Deferred@970796492 after retry RPC@241792825) -> (continuation of Deferred@81239517 after retry RPC@1103291073) -> (continuation of Deferred@1337715392 after retry RPC@423218460), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@69c668e7 -> wakeup thread Time-limited test -> (continuation of Deferred@461503964 after retry RPC@1096429339) -> (continuation of Deferred@1153792023 after retry RPC@1417127267) -> (continuation of Deferred@1382631143 after retry RPC@167239312) -> (continuation of Deferred@970796492 after retry RPC@241792825) -> (continuation of Deferred@81239517 after retry RPC@1103291073) -> (continuation of Deferred@1337715392 after retry RPC@423218460))) after lookup
21:08:17.258 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:17.259 [DEBUG - kudu-nio-0] (Connection.java:757) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] Successfully connected to peer
21:08:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.261235 18436 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 9 FOLLOWER]: Refusing update from remote peer c44178f4b5464e1aa2934204708c52a1: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
21:08:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.261744 18496 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
21:08:17.277 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc978fb9c, L:/127.0.0.1:39670 - R:/127.16.75.66:44849] using TOKEN/null
21:08:17.288 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.67:41157
21:08:17.288 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.288952 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.372218 18525 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.372471 18525 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.372507 18525 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.374447 18525 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.374550 18525 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.67
21:08:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.376616 18525 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18525
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:17.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.376947 18525 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.377242 18525 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.380126 18530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.380141 18531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:17.380141 18533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.380414 18525 server_base.cc:1061] running on GCE node
21:08:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.380566 18525 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.380734 18525 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.381896 18525 hybrid_clock.cc:648] HybridClock initialized: now 1783544897381870 us; error 34 us; skew 500 ppm
21:08:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.383134 18525 webserver.cc:534] Webserver started at http://127.16.75.67:39433/ using document root <none> and password file <none>
21:08:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.383379 18525 fs_manager.cc:362] Metadata directory not provided
21:08:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.383481 18525 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.384882 18525 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
21:08:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.385653 18539 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.385924 18525 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.386008 18525 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.386369 18525 fs_report.cc:389] FS layout report
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/data
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.410531 18525 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.410849 18525 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.410938 18525 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.411157 18525 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.411594 18546 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.412599 18525 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.412667 18525 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.412698 18525 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.413429 18525 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.413475 18525 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.413574 18546 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap starting.
21:08:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.421432 18525 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.67:41157
21:08:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.421504 18653 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.67:41157 every 8 connection(s)
21:08:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.421865 18525 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:17.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.423982 18546 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Log is configured to *not* fsync() on all Append() calls
21:08:17.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.425012 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 18525
21:08:17.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.428388 18546 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.428767 18546 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap complete.
21:08:17.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.429888 18546 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent bootstrapping tablet: real 0.016s	user 0.009s	sys 0.006s
21:08:17.430 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:17.431 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.66:44849
21:08:17.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:08:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.431162 18546 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.431537 18546 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Initialized, Role: FOLLOWER
21:08:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.431727 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 18167
21:08:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.431704 18546 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.433997 18655 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434120 18655 heartbeater.cc:461] Registering TS with master...
21:08:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434139 18656 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434209 18546 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent starting tablet: real 0.004s	user 0.001s	sys 0.002s
21:08:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434216 18656 heartbeater.cc:461] Registering TS with master...
21:08:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434341 18655 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434365 18656 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434955 16798 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.434955 16726 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.435351 18654 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.435413 18654 heartbeater.cc:461] Registering TS with master...
21:08:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.435451 16798 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.67:42181
21:08:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.435505 18654 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.435992 16871 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:17.436234 18656 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:17.439 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:17.439 [DEBUG - kudu-nio-0] (Connection.java:239) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] handling channelInactive
21:08:17.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state READY due to: connection closed
21:08:17.455 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:17.457 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.457 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.457 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.477 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:17.477 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:17.477 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=3, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [18ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [18ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [37ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@356137510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@402f28a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d060bda -> wakeup thread Time-limited test)) after lookup
21:08:17.478 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:17.479 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.479 [DEBUG - kudu-nio-2] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.479 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.497 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:17.497 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:17.498 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=5, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [18ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [18ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [37ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [39ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [40ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [40ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@356137510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@402f28a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d060bda -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147))) after lookup
21:08:17.498 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:17.499 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.499 [DEBUG - kudu-nio-3] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.500 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.517 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:17.517 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:17.518 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=7, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [18ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [18ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [37ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [39ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [40ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [40ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [61ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [61ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@356137510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@402f28a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d060bda -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283))) after lookup
21:08:17.519 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:17.520 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.520 [DEBUG - kudu-nio-4] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.520 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.697 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:17.697 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:17.698 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=9, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=258), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [18ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [18ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [37ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [39ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [40ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [40ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [61ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [61ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [79ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [81ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [81ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [255ms] refreshing cache from master, [255ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [257ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@356137510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@402f28a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283) -> (continuation of Deferred@9594643 after retry RPC@1639909664), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d060bda -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283) -> (continuation of Deferred@9594643 after retry RPC@1639909664))) after lookup
21:08:17.698 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:17.700 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.700 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:17.700 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:18.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:18.377 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:18.378 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=11, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=938), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [18ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [18ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [37ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [39ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [40ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [40ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [61ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [61ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [79ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [81ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [81ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [255ms] refreshing cache from master, [255ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [257ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [259ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [261ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [261ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [936ms] refreshing cache from master, [936ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [938ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@356137510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@402f28a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283) -> (continuation of Deferred@9594643 after retry RPC@1639909664) -> (continuation of Deferred@1408956188 after retry RPC@1251283530), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d060bda -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283) -> (continuation of Deferred@9594643 after retry RPC@1639909664) -> (continuation of Deferred@1408956188 after retry RPC@1251283530))) after lookup
21:08:18.378 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:18.380 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:18.380 [DEBUG - kudu-nio-6] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state CONNECTING due to: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:18.380 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849
21:08:18.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.787550 18667 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader c44178f4b5464e1aa2934204708c52a1)
21:08:18.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.787657 18667 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:18.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.788025 18667 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:18.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:18.788508 18368 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111)
21:08:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:18.788992 18368 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111)
21:08:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.791386 18608 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" is_pre_election: true
21:08:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.791553 18608 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 62a04151dfcd4eaf9bc55e4985fac600 in term 8.
21:08:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.791893 18369 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748; no voters: c44178f4b5464e1aa2934204708c52a1
21:08:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.792033 18667 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 9 FOLLOWER]: Leader pre-election won for term 10
21:08:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.792089 18667 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 9 FOLLOWER]: Starting leader election (detected failure of leader c44178f4b5464e1aa2934204708c52a1)
21:08:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.792105 18667 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 9 FOLLOWER]: Advancing to term 10
21:08:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.792692 18667 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.792840 18667 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 10 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.793015 18608 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:08:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.793115 18608 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 8 FOLLOWER]: Advancing to term 10
21:08:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:18.793442 18368 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111)
21:08:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.794332 18608 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 62a04151dfcd4eaf9bc55e4985fac600 in term 10.
21:08:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.794543 18369 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, 6c0e3990f3964f1e9b57977ffa9d6748; no voters: c44178f4b5464e1aa2934204708c52a1
21:08:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.794637 18667 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Leader election won for term 10
21:08:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.794766 18667 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 LEADER]: Becoming Leader. State: Replica: 62a04151dfcd4eaf9bc55e4985fac600, State: Running, Role: LEADER
21:08:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.794875 18667 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:18.795626 16798 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 reported cstate change: term changed from 9 to 10, leader changed from c44178f4b5464e1aa2934204708c52a1 (127.16.75.66) to 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65). New cstate: current_term: 10 leader_uuid: "62a04151dfcd4eaf9bc55e4985fac600" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: UNKNOWN } } }
21:08:19.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:19.288834 18368 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 -> Peer c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849): Couldn't send request to peer c44178f4b5464e1aa2934204708c52a1. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.66:44849: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:19.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:19.292220 18608 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 10 FOLLOWER]: Refusing update from remote peer 62a04151dfcd4eaf9bc55e4985fac600: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 33. (index mismatch)
21:08:19.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:19.292594 18667 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
21:08:21.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:21.457 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:21.458 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=13, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=4018), Traces: [0ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [0ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [18ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [18ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [37ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [39ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [40ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [40ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [58ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [59ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [61ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [61ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [79ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [81ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [81ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [255ms] refreshing cache from master, [255ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [257ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [259ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [261ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [261ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [936ms] refreshing cache from master, [936ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [938ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [939ms] sending RPC to server c44178f4b5464e1aa2934204708c52a1, [941ms] received response from server c44178f4b5464e1aa2934204708c52a1: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [941ms] delaying RPC due to: Network error: Failed to connect to peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849): Connection refused: /127.16.75.66:44849, [4016ms] refreshing cache from master, [4016ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [4018ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@356137510(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@402f28a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283) -> (continuation of Deferred@9594643 after retry RPC@1639909664) -> (continuation of Deferred@1408956188 after retry RPC@1251283530) -> (continuation of Deferred@1854155728 after retry RPC@207708936), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6d060bda -> wakeup thread Time-limited test -> (continuation of Deferred@935033773 after retry RPC@1782509147) -> (continuation of Deferred@1028212567 after retry RPC@546843283) -> (continuation of Deferred@9594643 after retry RPC@1639909664) -> (continuation of Deferred@1408956188 after retry RPC@1251283530) -> (continuation of Deferred@1854155728 after retry RPC@207708936))) after lookup
21:08:21.458 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:21.460 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] Successfully connected to peer
21:08:21.485 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x9c8c4792, L:/127.0.0.1:56484 - R:/127.16.75.65:37515] using TOKEN/null
21:08:21.496 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.66:44849
21:08:21.496 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.497170 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:44849
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:21.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.579021 18707 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:21.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.579232 18707 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:21.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.579254 18707 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.580875 18707 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.580962 18707 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.66
21:08:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.582679 18707 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.66:44849
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.66
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18707
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.66
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/logs
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.582968 18707 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.583173 18707 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.586544 18707 server_base.cc:1061] running on GCE node
21:08:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.586483 18713 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.586481 18715 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:21.586503 18712 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.586862 18707 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.587074 18707 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:21.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.588209 18707 hybrid_clock.cc:648] HybridClock initialized: now 1783544901588195 us; error 34 us; skew 500 ppm
21:08:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.589576 18707 webserver.cc:534] Webserver started at http://127.16.75.66:39729/ using document root <none> and password file <none>
21:08:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.589905 18707 fs_manager.cc:362] Metadata directory not provided
21:08:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.589972 18707 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.591421 18707 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
21:08:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.592146 18721 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.592398 18707 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.000s
21:08:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.592482 18707 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c44178f4b5464e1aa2934204708c52a1"
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.592795 18707 fs_report.cc:389] FS layout report
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/wal
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/data
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.604161 18707 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.604498 18707 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.604597 18707 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.604785 18707 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.605211 18728 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.606190 18707 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.606244 18707 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.606272 18707 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.606802 18707 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.606834 18707 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.606966 18728 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap starting.
21:08:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.614892 18707 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.66:44849
21:08:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.615000 18835 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.66:44849 every 8 connection(s)
21:08:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.615434 18707 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-1/data/info.pb
21:08:21.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.617106 18728 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Log is configured to *not* fsync() on all Append() calls
21:08:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622148 18837 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622165 18728 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622294 18837 heartbeater.cc:461] Registering TS with master...
21:08:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622421 18836 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622468 18836 heartbeater.cc:461] Registering TS with master...
21:08:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622552 18837 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622551 18836 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:21.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622944 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 18707
21:08:21.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.622572 18728 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Bootstrap complete.
21:08:21.623 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:21.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.623431 16871 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.623934 16726 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.624226 18838 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.624298 18838 heartbeater.cc:461] Registering TS with master...
21:08:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.624354 18728 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent bootstrapping tablet: real 0.017s	user 0.007s	sys 0.008s
21:08:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.624436 18838 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:21.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.624879 16799 ts_manager.cc:194] Re-registered known tserver with Master: c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:21.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.625411 16799 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.66:57673
21:08:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.625689 18728 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.625909 18728 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Initialized, Role: FOLLOWER
21:08:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.626031 18728 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.626307 18728 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1: Time spent starting tablet: real 0.002s	user 0.001s	sys 0.001s
21:08:21.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.628170 18838 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:21.628 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:21.629 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.65:37515
21:08:21.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:08:21.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:21.629917 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 18353
21:08:21.637 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] handling channelInactive
21:08:21.637 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state READY due to: connection closed
21:08:21.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:21.638 [DEBUG - Time-limited test] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:21.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.639 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.657 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:21.657 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:21.658 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1374480072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@5612b6de -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44b5cc36 -> wakeup thread Time-limited test)) after lookup
21:08:21.658 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:21.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.659 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.677 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:21.679 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:21.679 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [20ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [21ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [21ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1374480072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@5612b6de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44b5cc36 -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546))) after lookup
21:08:21.680 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:21.681 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.681 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.681 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.717 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:21.717 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:21.717 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [20ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [21ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [21ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [41ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1374480072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@5612b6de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44b5cc36 -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276))) after lookup
21:08:21.718 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:21.719 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.719 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.719 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.757 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:21.757 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:21.758 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=119), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [20ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [21ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [21ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [41ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [79ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [81ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [81ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [119ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1374480072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@5612b6de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276) -> (continuation of Deferred@1333655484 after retry RPC@2015345961), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44b5cc36 -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276) -> (continuation of Deferred@1333655484 after retry RPC@2015345961))) after lookup
21:08:21.758 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:21.759 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.759 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:21.759 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:22.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:22.257 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)[L],6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:22.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=619), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [20ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [21ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [21ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [41ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [79ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [81ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [81ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [119ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [120ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [121ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [121ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [616ms] refreshing cache from master, [617ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [619ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1374480072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@5612b6de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276) -> (continuation of Deferred@1333655484 after retry RPC@2015345961) -> (continuation of Deferred@1429315983 after retry RPC@1346550005), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44b5cc36 -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276) -> (continuation of Deferred@1333655484 after retry RPC@2015345961) -> (continuation of Deferred@1429315983 after retry RPC@1346550005))) after lookup
21:08:22.258 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] connecting to peer
21:08:22.259 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:22.259 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] cleaning up while in state CONNECTING due to: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:22.259 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515
21:08:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:22.995780 18849 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 62a04151dfcd4eaf9bc55e4985fac600)
21:08:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:22.995939 18849 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:22.996340 18849 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:22.996780 18543 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:22.997318 18543 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:22.999979 18790 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1" is_pre_election: true
21:08:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.000133 18790 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 9.
21:08:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.000391 18540 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600
21:08:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.000562 18849 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 10 FOLLOWER]: Leader pre-election won for term 11
21:08:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.000622 18849 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 62a04151dfcd4eaf9bc55e4985fac600)
21:08:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.000638 18849 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 10 FOLLOWER]: Advancing to term 11
21:08:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.001207 18849 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.001379 18849 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 11 election: Requested vote from peers 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.001575 18790 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1"
21:08:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.001667 18790 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 9 FOLLOWER]: Advancing to term 11
21:08:23.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:23.001996 18543 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111)
21:08:23.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.002786 18790 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6c0e3990f3964f1e9b57977ffa9d6748 in term 11.
21:08:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.002965 18540 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1; no voters: 62a04151dfcd4eaf9bc55e4985fac600
21:08:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.003086 18849 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 11 FOLLOWER]: Leader election won for term 11
21:08:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.003268 18849 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 11 LEADER]: Becoming Leader. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Running, Role: LEADER
21:08:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.003417 18849 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.004238 16799 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 reported cstate change: term changed from 10 to 11, leader changed from 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65) to 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67). New cstate: current_term: 11 leader_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: UNKNOWN } } }
21:08:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.430049 18790 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 11 FOLLOWER]: Refusing update from remote peer 6c0e3990f3964f1e9b57977ffa9d6748: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 37. (index mismatch)
21:08:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:23.430469 18849 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
21:08:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:23.433633 18543 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 -> Peer 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515): Couldn't send request to peer 62a04151dfcd4eaf9bc55e4985fac600. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.65:37515: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:23.957 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:23.957 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:23.958 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2320), Traces: [0ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [1ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [1ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [19ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [20ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [21ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [21ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [39ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [41ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [43ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [43ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [78ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [79ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [81ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [81ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [119ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [120ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [121ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [121ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [616ms] refreshing cache from master, [617ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [619ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [620ms] sending RPC to server 62a04151dfcd4eaf9bc55e4985fac600, [621ms] received response from server 62a04151dfcd4eaf9bc55e4985fac600: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [621ms] delaying RPC due to: Network error: Failed to connect to peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515): Connection refused: /127.16.75.65:37515, [2316ms] refreshing cache from master, [2316ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [2318ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@1374480072(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@5612b6de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276) -> (continuation of Deferred@1333655484 after retry RPC@2015345961) -> (continuation of Deferred@1429315983 after retry RPC@1346550005) -> (continuation of Deferred@450521489 after retry RPC@1595604783), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@44b5cc36 -> wakeup thread Time-limited test -> (continuation of Deferred@1948091660 after retry RPC@1547990546) -> (continuation of Deferred@1533636453 after retry RPC@618817276) -> (continuation of Deferred@1333655484 after retry RPC@2015345961) -> (continuation of Deferred@1429315983 after retry RPC@1346550005) -> (continuation of Deferred@450521489 after retry RPC@1595604783))) after lookup
21:08:23.959 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:23.960 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] Successfully connected to peer
21:08:23.989 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xa804bdea, L:/127.0.0.1:45240 - R:/127.16.75.67:41157] using TOKEN/null
21:08:24.000 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.65:37515
21:08:24.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.001274 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:08:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:37515
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46433
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.085947 18883 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.086190 18883 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.086225 18883 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.087978 18883 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.088088 18883 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.65
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.089890 18883 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.65:37515
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.65
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46433
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18883
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.65
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/logs
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.090237 18883 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.090570 18883 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.093614 18891 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.093564 18890 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:24.093566 18893 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.093915 18883 server_base.cc:1061] running on GCE node
21:08:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.094055 18883 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.094326 18883 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:24.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.095521 18883 hybrid_clock.cc:648] HybridClock initialized: now 1783544904095487 us; error 41 us; skew 500 ppm
21:08:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.096984 18883 webserver.cc:534] Webserver started at http://127.16.75.65:46433/ using document root <none> and password file <none>
21:08:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.097259 18883 fs_manager.cc:362] Metadata directory not provided
21:08:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.097368 18883 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:24.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.098767 18883 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.002s	sys 0.000s
21:08:24.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.099558 18899 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:24.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.099779 18883 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.099952 18883 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:55 on dist-test-slave-0c79"
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.100309 18883 fs_report.cc:389] FS layout report
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/wal
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/data
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:24.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:24.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.119804 18883 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:24.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.120265 18883 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:24.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.120415 18883 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:24.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.120687 18883 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.121213 18906 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.122232 18883 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.122308 18883 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.122345 18883 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.122937 18883 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.122994 18883 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.123042 18906 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap starting.
21:08:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.130496 18883 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.65:37515
21:08:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.130605 19013 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.65:37515 every 8 connection(s)
21:08:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.130957 18883 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-0/data/info.pb
21:08:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.131314 18906 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Log is configured to *not* fsync() on all Append() calls
21:08:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.136531 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 18883
21:08:24.137 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.136940 18906 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.137290 18906 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Bootstrap complete.
21:08:24.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.138392 18906 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent bootstrapping tablet: real 0.015s	user 0.006s	sys 0.006s
21:08:24.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.139873 18906 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:24.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.140168 18906 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62a04151dfcd4eaf9bc55e4985fac600, State: Initialized, Role: FOLLOWER
21:08:24.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.140278 18906 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:24.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.140534 18906 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600: Time spent starting tablet: real 0.002s	user 0.005s	sys 0.000s
21:08:24.141 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.141481 19014 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.141505 19015 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.141592 19014 heartbeater.cc:461] Registering TS with master...
21:08:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.141593 19015 heartbeater.cc:461] Registering TS with master...
21:08:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.141816 19015 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:24.142 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.16.75.67:41157
21:08:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.141816 19014 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.142454 16726 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:24.142 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:08:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.142871 16871 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.142937 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 18525
21:08:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.148347 19016 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.148455 19016 heartbeater.cc:461] Registering TS with master...
21:08:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.148588 19016 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.148872 16799 ts_manager.cc:194] Re-registered known tserver with Master: 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:24.149 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] handling channelInactive
21:08:24.150 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state READY due to: connection closed
21:08:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.149260 16799 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.65:59177
21:08:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:24.150126 19016 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:24.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:24.151 [DEBUG - Time-limited test] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:24.152 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.152 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.152 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.156 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:24.156 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:24.157 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=6), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [5ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@819741087(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@38b7332c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a1668ca -> wakeup thread Time-limited test)) after lookup
21:08:24.157 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:24.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.158 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.177 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:24.177 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:24.178 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [5ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [6ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [7ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [7ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@819741087(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@38b7332c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a1668ca -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943))) after lookup
21:08:24.178 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:24.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.179 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.217 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:24.217 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:24.217 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [5ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [6ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [7ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [7ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [27ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [28ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [65ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@819741087(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@38b7332c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a1668ca -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931))) after lookup
21:08:24.218 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:24.219 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.219 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.219 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:24.257 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:24.257 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=106), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [5ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [6ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [7ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [7ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [27ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [28ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [65ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [66ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [68ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [68ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [105ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@819741087(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@38b7332c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931) -> (continuation of Deferred@721187352 after retry RPC@1703761017), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a1668ca -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931) -> (continuation of Deferred@721187352 after retry RPC@1703761017))) after lookup
21:08:24.258 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:24.259 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.259 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.259 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:24.457 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)[L],c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)]}]
21:08:24.457 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=306), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [5ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [6ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [7ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [7ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [27ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [28ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [65ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [66ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [68ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [68ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [105ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [107ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [108ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [108ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [303ms] refreshing cache from master, [303ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [305ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@819741087(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@38b7332c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931) -> (continuation of Deferred@721187352 after retry RPC@1703761017) -> (continuation of Deferred@155477565 after retry RPC@115721712), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a1668ca -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931) -> (continuation of Deferred@721187352 after retry RPC@1703761017) -> (continuation of Deferred@155477565 after retry RPC@115721712))) after lookup
21:08:24.458 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] connecting to peer
21:08:24.459 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.459 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:24.459 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157) for tablet 51618a18fd6f49398ed01f49b19c347f: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157
21:08:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.294008 19027 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:08:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.294117 19027 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.294467 19027 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), c44178f4b5464e1aa2934204708c52a1 (127.16.75.66:44849)
21:08:25.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:25.294911 18901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:25.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:25.295377 18901 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.297880 18790 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "62a04151dfcd4eaf9bc55e4985fac600" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "c44178f4b5464e1aa2934204708c52a1" is_pre_election: true
21:08:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.298177 18900 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600; no voters: 6c0e3990f3964f1e9b57977ffa9d6748, c44178f4b5464e1aa2934204708c52a1
21:08:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.298326 19027 raft_consensus.cc:2749] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Leader pre-election lost for term 11. Reason: could not achieve majority
21:08:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.500545 19032 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:08:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.500649 19032 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.500914 19032 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:25.501425 18723 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:25.501991 18723 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.504236 18968 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600" is_pre_election: true
21:08:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.504375 18968 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 10.
21:08:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.504645 18725 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:08:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.504815 19032 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 11 FOLLOWER]: Leader pre-election won for term 12
21:08:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.504876 19032 raft_consensus.cc:493] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 11 FOLLOWER]: Starting leader election (detected failure of leader 6c0e3990f3964f1e9b57977ffa9d6748)
21:08:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.504892 19032 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 11 FOLLOWER]: Advancing to term 12
21:08:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.505465 19032 raft_consensus.cc:515] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.505620 19032 leader_election.cc:290] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 12 election: Requested vote from peers 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157), 62a04151dfcd4eaf9bc55e4985fac600 (127.16.75.65:37515)
21:08:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.505810 18968 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "51618a18fd6f49398ed01f49b19c347f" candidate_uuid: "c44178f4b5464e1aa2934204708c52a1" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "62a04151dfcd4eaf9bc55e4985fac600"
21:08:25.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.505892 18968 raft_consensus.cc:3060] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 10 FOLLOWER]: Advancing to term 12
21:08:25.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:25.506276 18723 leader_election.cc:336] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111)
21:08:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.506981 18968 raft_consensus.cc:2468] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c44178f4b5464e1aa2934204708c52a1 in term 12.
21:08:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.507190 18725 leader_election.cc:304] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62a04151dfcd4eaf9bc55e4985fac600, c44178f4b5464e1aa2934204708c52a1; no voters: 6c0e3990f3964f1e9b57977ffa9d6748
21:08:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.507335 19032 raft_consensus.cc:2804] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 12 FOLLOWER]: Leader election won for term 12
21:08:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.507479 19032 raft_consensus.cc:697] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [term 12 LEADER]: Becoming Leader. State: Replica: c44178f4b5464e1aa2934204708c52a1, State: Running, Role: LEADER
21:08:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.507596 19032 consensus_queue.cc:237] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:25.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:25.508415 16799 catalog_manager.cc:5696] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 reported cstate change: term changed from 11 to 12, leader changed from 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67) to c44178f4b5464e1aa2934204708c52a1 (127.16.75.66). New cstate: current_term: 12 leader_uuid: "c44178f4b5464e1aa2934204708c52a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } health_report { overall_health: HEALTHY } } }
21:08:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.083053 18723 consensus_peers.cc:597] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 -> Peer 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157): Couldn't send request to peer 6c0e3990f3964f1e9b57977ffa9d6748. Status: Network error: Client connection negotiation failed: client connection to 127.16.75.67:41157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:08:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.083408 18968 raft_consensus.cc:1275] T 51618a18fd6f49398ed01f49b19c347f P 62a04151dfcd4eaf9bc55e4985fac600 [term 12 FOLLOWER]: Refusing update from remote peer c44178f4b5464e1aa2934204708c52a1: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 41. (index mismatch)
21:08:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.083740 19049 consensus_queue.cc:1048] T 51618a18fd6f49398ed01f49b19c347f P c44178f4b5464e1aa2934204708c52a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
21:08:26.437 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 51618a18fd6f49398ed01f49b19c347f for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718' with partition [<start>, <end>)
21:08:26.437 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=51618a18fd6f49398ed01f49b19c347f@[62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515),6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157),c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)[L]]}]
21:08:26.437 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=51618a18fd6f49398ed01f49b19c347f, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2286), Traces: [0ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [1ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [3ms] refreshing cache from master, [3ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [5ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [6ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [7ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [7ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [26ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [27ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [28ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [28ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [63ms] refreshing cache from master, [64ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [65ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [66ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [68ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [68ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [105ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [107ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [108ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [108ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [303ms] refreshing cache from master, [303ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [305ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, [307ms] sending RPC to server 6c0e3990f3964f1e9b57977ffa9d6748, [308ms] received response from server 6c0e3990f3964f1e9b57977ffa9d6748: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [308ms] delaying RPC due to: Network error: Failed to connect to peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157): Connection refused: /127.16.75.67:41157, [2283ms] refreshing cache from master, [2284ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.75.125:36119, [2286ms] Sub RPC GetTableLocations: received response from server master-127.16.75.125:36119: OK, deferred=Deferred@819741087(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$231/0x00007fae5c36acb0@38b7332c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931) -> (continuation of Deferred@721187352 after retry RPC@1703761017) -> (continuation of Deferred@155477565 after retry RPC@115721712) -> (continuation of Deferred@1082576028 after retry RPC@1751403321), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@a1668ca -> wakeup thread Time-limited test -> (continuation of Deferred@957803423 after retry RPC@8356943) -> (continuation of Deferred@1825347379 after retry RPC@1402176931) -> (continuation of Deferred@721187352 after retry RPC@1703761017) -> (continuation of Deferred@155477565 after retry RPC@115721712) -> (continuation of Deferred@1082576028 after retry RPC@1751403321))) after lookup
21:08:26.438 [DEBUG - kudu-nio-1] (Connection.java:748) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] connecting to peer
21:08:26.439 [DEBUG - kudu-nio-5] (Connection.java:757) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] Successfully connected to peer
21:08:26.461 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0xba041d67, L:/127.0.0.1:39452 - R:/127.16.75.66:44849] using TOKEN/null
21:08:26.473 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.16.75.67:41157
21:08:26.474 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.474329 16685 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDb8yoT/build/release/bin/kudu
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:08:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:08:26.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.555270 19063 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:08:26.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.555470 19063 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:08:26.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.555492 19063 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:08:26.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.557209 19063 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:08:26.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.557300 19063 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.75.67
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.559048 19063 tablet_server_runner.cc:78] Tablet server non-default flags:
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.75.84:45695
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.75.67:41157
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.75.67
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39433
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.75.126:34341,127.16.75.125:36119,127.16.75.124:37917
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.19063
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.75.67
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/logs
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:08:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:08:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6bec869c2320d19f2589b3f609f76e5041a21d27
21:08:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
21:08:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 20:43:47 UTC on e7f111948823
21:08:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16876
21:08:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.559298 19063 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:08:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.559494 19063 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:08:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.562561 19071 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.562610 19068 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 21:08:26.562561 19069 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.563019 19063 server_base.cc:1061] running on GCE node
21:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.563201 19063 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:08:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.563412 19063 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:08:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.564591 19063 hybrid_clock.cc:648] HybridClock initialized: now 1783544906564561 us; error 36 us; skew 500 ppm
21:08:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.566056 19063 webserver.cc:534] Webserver started at http://127.16.75.67:39433/ using document root <none> and password file <none>
21:08:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.566350 19063 fs_manager.cc:362] Metadata directory not provided
21:08:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.566462 19063 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:08:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.568128 19063 fs_manager.cc:714] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.568962 19077 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.569201 19063 fs_manager.cc:730] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.569280 19063 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data,/tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c0e3990f3964f1e9b57977ffa9d6748"
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 21:07:56 on dist-test-slave-0c79"
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.569680 19063 fs_report.cc:389] FS layout report
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:08:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/wal
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/data
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:08:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:08:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.582301 19063 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:08:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.582695 19063 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:08:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.582880 19063 kserver.cc:163] Server-wide thread pool size limit: 3276
21:08:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.583161 19063 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:08:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.583695 19084 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
21:08:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.584602 19063 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
21:08:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.584671 19063 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
21:08:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.584726 19063 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
21:08:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.585285 19063 ts_tablet_manager.cc:616] Registered 1 tablets
21:08:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.585346 19063 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
21:08:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.585374 19084 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap starting.
21:08:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.592684 19063 rpc_server.cc:307] RPC server started. Bound to: 127.16.75.67:41157
21:08:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.592788 19191 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.75.67:41157 every 8 connection(s)
21:08:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.593269 19063 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskDb8yoT/test-tmp/mini-kudu-cluster504105440528584215/ts-2/data/info.pb
21:08:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.596374 19084 log.cc:826] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Log is configured to *not* fsync() on all Append() calls
21:08:26.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.599331 16685 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskDb8yoT/build/release/bin/kudu as pid 19063
21:08:26.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:08:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.600063 19193 heartbeater.cc:344] Connected to a master server at 127.16.75.126:34341
21:08:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.600183 19193 heartbeater.cc:461] Registering TS with master...
21:08:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.600409 19193 heartbeater.cc:507] Master 127.16.75.126:34341 requested a full tablet report, sending...
21:08:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.600890 19192 heartbeater.cc:344] Connected to a master server at 127.16.75.124:37917
21:08:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.600950 19192 heartbeater.cc:461] Registering TS with master...
21:08:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.601051 19192 heartbeater.cc:507] Master 127.16.75.124:37917 requested a full tablet report, sending...
21:08:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.601769 16871 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.601872 19084 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:08:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.602226 19084 tablet_bootstrap.cc:492] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Bootstrap complete.
21:08:26.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.602972 19194 heartbeater.cc:344] Connected to a master server at 127.16.75.125:36119
21:08:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.603044 19194 heartbeater.cc:461] Registering TS with master...
21:08:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.603145 19194 heartbeater.cc:507] Master 127.16.75.125:36119 requested a full tablet report, sending...
21:08:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.603149 16726 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.603304 19084 ts_tablet_manager.cc:1403] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent bootstrapping tablet: real 0.018s	user 0.004s	sys 0.010s
21:08:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.603412 16799 ts_manager.cc:194] Re-registered known tserver with Master: 6c0e3990f3964f1e9b57977ffa9d6748 (127.16.75.67:41157)
21:08:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.603932 16799 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.75.67:49589
21:08:26.605 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:26.608 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783544876718, tablet=51618a18fd6f49398ed01f49b19c347f, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
21:08:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.604537 19084 raft_consensus.cc:359] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.604809 19084 raft_consensus.cc:740] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c0e3990f3964f1e9b57977ffa9d6748, State: Initialized, Role: FOLLOWER
21:08:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.604938 19084 consensus_queue.cc:260] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c0e3990f3964f1e9b57977ffa9d6748" member_type: VOTER last_known_addr { host: "127.16.75.67" port: 41157 } } peers { permanent_uuid: "62a04151dfcd4eaf9bc55e4985fac600" member_type: VOTER last_known_addr { host: "127.16.75.65" port: 37515 } } peers { permanent_uuid: "c44178f4b5464e1aa2934204708c52a1" member_type: VOTER last_known_addr { host: "127.16.75.66" port: 44849 } }
21:08:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.605233 19084 ts_tablet_manager.cc:1434] T 51618a18fd6f49398ed01f49b19c347f P 6c0e3990f3964f1e9b57977ffa9d6748: Time spent starting tablet: real 0.002s	user 0.005s	sys 0.000s
21:08:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.605779 19194 heartbeater.cc:499] Master 127.16.75.125:36119 was elected leader, sending a full tablet report...
21:08:26.615 [DEBUG - main] (Connection.java:575) [peer master-127.16.75.126:34341(127.16.75.126:34341)] disconnecting while in state READY
21:08:26.617 [DEBUG - main] (Connection.java:575) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] disconnecting while in state READY
21:08:26.617 [DEBUG - main] (Connection.java:575) [peer master-127.16.75.125:36119(127.16.75.125:36119)] disconnecting while in state READY
21:08:26.618 [DEBUG - main] (Connection.java:575) [peer 6c0e3990f3964f1e9b57977ffa9d6748(127.16.75.67:41157)] disconnecting while in state TERMINATED
21:08:26.619 [DEBUG - main] (Connection.java:575) [peer master-127.16.75.124:37917(127.16.75.124:37917)] disconnecting while in state READY
21:08:26.619 [DEBUG - main] (Connection.java:575) [peer 62a04151dfcd4eaf9bc55e4985fac600(127.16.75.65:37515)] disconnecting while in state TERMINATED
21:08:26.620 [DEBUG - kudu-nio-5] (Connection.java:239) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] handling channelInactive
21:08:26.620 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.16.75.124:37917(127.16.75.124:37917)] handling channelInactive
21:08:26.620 [DEBUG - kudu-nio-5] (Connection.java:709) [peer c44178f4b5464e1aa2934204708c52a1(127.16.75.66:44849)] cleaning up while in state READY due to: connection closed
21:08:26.620 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.75.124:37917(127.16.75.124:37917)] cleaning up while in state READY due to: connection closed
21:08:26.620 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.75.126:34341(127.16.75.126:34341)] handling channelInactive
21:08:26.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.75.126:34341(127.16.75.126:34341)] cleaning up while in state READY due to: connection closed
21:08:26.620 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:08:26.621 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.75.125:36119(127.16.75.125:36119)] handling channelInactive
21:08:26.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.75.125:36119(127.16.75.125:36119)] cleaning up while in state READY due to: connection closed
21:08:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.623939 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 18883
21:08:26.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.629900 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 18707
21:08:26.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.635941 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 19063
21:08:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.641388 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 16696
21:08:26.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.646835 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 16767
21:08:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 21:08:26.652186 16685 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskDb8yoT/build/release/bin/kudu with pid 16838
21:08:26.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T21:08:26Z chronyd exiting

Time: 32.324

OK (1 test)

