JUnit version 4.13.2
.18:51:10.817 [INFO - main] (RandomUtils.java:49) Using random seed: 1786474270809
18:51:10.863 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
18:51:10.923 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
18:51:10.938 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskIxmF87/test-tmp
18:51:10.941 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
18:51:10.945 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskIxmF87/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
18:51:11.168 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

18:51:11.187 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:51:11.188 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

18:51:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.189553 28702 mini_oidc.cc:152] Starting JWKS server
18:51:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.190650 28702 webserver.cc:460] Webserver started at http://127.0.0.1:38211/ using document root <none> and password file <none>
18:51:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.190745 28702 mini_oidc.cc:189] Starting OIDC Discovery server
18:51:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.190940 28702 webserver.cc:460] Webserver started at http://127.0.0.1:46475/ using document root <none> and password file <none>
18:51:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T18:51:11Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
18:51:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T18:51:11Z Disabled control of system clock
18:51:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.205345 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/wal
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/logs
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data/info.pb
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.190:38569
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.190
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:51:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:51:11.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38211/jwks.json with env {}
18:51:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.306928 28715 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.307168 28715 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.307191 28715 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.308619 28715 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:51:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.308691 28715 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.308713 28715 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:51:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.308732 28715 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:51:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.308749 28715 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:51:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.308766 28715 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38211/jwks.json
18:51:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.310930 28715 master_runner.cc:386] Master server non-default flags:
18:51:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data
18:51:11.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/wal
18:51:11.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.190:38569
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38211/jwks.json
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data/info.pb
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.190
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.28715
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:11.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/logs
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:11.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.311252 28715 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:11.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.311631 28715 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:11.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.318465 28726 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.318506 28723 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.318638 28715 server_base.cc:1034] running on GCE node
18:51:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.318463 28722 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.319052 28715 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.319427 28715 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:11.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.320596 28715 hybrid_clock.cc:648] HybridClock initialized: now 1786474271320595 us; error 107 us; skew 500 ppm
18:51:11.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.322078 28715 webserver.cc:460] Webserver started at http://127.28.7.190:40011/ using document root <none> and password file <none>
18:51:11.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.322372 28715 fs_manager.cc:359] Metadata directory not provided
18:51:11.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.322464 28715 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:11.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.322613 28715 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:51:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.323717 28715 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data/instance:
18:51:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3487e6c7dce54f27bd27a4efdb1947ce"
18:51:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.324142 28715 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/wal/instance:
18:51:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3487e6c7dce54f27bd27a4efdb1947ce"
18:51:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.325929 28715 fs_manager.cc:691] Time spent creating directory manager: real 0.002s	user 0.000s	sys 0.004s
18:51:11.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.326849 28733 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:11.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.327045 28715 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:51:11.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.327138 28715 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/wal
18:51:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3487e6c7dce54f27bd27a4efdb1947ce"
18:51:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.327207 28715 fs_report.cc:383] FS layout report
18:51:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/wal
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/wal
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data/data
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.350991 28715 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.351424 28715 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.351598 28715 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.355749 28715 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.190:38569
18:51:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.355790 28795 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.190:38569 every 8 connection(s)
18:51:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.356240 28715 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/data/info.pb
18:51:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.357122 28797 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:51:11.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.358987 28797 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.363282 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 28715
18:51:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.363431 28702 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-0/wal/instance
18:51:11.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.364681 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/wal
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/logs
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data/info.pb
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.189:39853
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.189
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:51:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:51:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38211/jwks.json with env {}
18:51:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.366169 28797 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.366705 28738 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.189:39853: connect: Connection refused (error 111)
18:51:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.367290 28797 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.7.189:39853: Network error: Client connection negotiation failed: client connection to 127.28.7.189:39853: connect: Connection refused (error 111)
18:51:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.416574 28797 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } attempt: 1
18:51:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.417573 28797 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.7.189:39853: Network error: Client connection negotiation failed: client connection to 127.28.7.189:39853: connect: Connection refused (error 111)
18:51:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.453310 28801 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.453528 28801 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.453549 28801 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.455019 28801 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:51:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.455076 28801 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.455090 28801 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:51:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.455103 28801 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:51:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.455116 28801 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:51:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.455127 28801 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38211/jwks.json
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.457232 28801 master_runner.cc:386] Master server non-default flags:
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/wal
18:51:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.189:39853
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38211/jwks.json
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data/info.pb
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.189
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.28801
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/logs
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.457545 28801 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.457940 28801 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.463476 28808 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.463496 28807 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.463541 28810 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.463845 28801 server_base.cc:1034] running on GCE node
18:51:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.464037 28801 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.464313 28801 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.465462 28801 hybrid_clock.cc:648] HybridClock initialized: now 1786474271465441 us; error 34 us; skew 500 ppm
18:51:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.466795 28801 webserver.cc:460] Webserver started at http://127.28.7.189:39935/ using document root <none> and password file <none>
18:51:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.467010 28801 fs_manager.cc:359] Metadata directory not provided
18:51:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.467052 28801 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.467144 28801 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:51:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.468214 28801 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data/instance:
18:51:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4b6180a702942fd9805fb8c3048953a"
18:51:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.468518 28801 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/wal/instance:
18:51:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4b6180a702942fd9805fb8c3048953a"
18:51:11.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.470206 28801 fs_manager.cc:691] Time spent creating directory manager: real 0.002s	user 0.000s	sys 0.003s
18:51:11.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.471165 28818 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.472934 28801 fs_manager.cc:725] Time spent opening block manager: real 0.002s	user 0.000s	sys 0.001s
18:51:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.473042 28801 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/wal
18:51:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4b6180a702942fd9805fb8c3048953a"
18:51:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.473114 28801 fs_report.cc:383] FS layout report
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/wal
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/wal
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data/data
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.484299 28801 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.484587 28801 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.484687 28801 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.485749 28797 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } attempt: 2
18:51:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.486709 28797 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.7.189:39853: Network error: Client connection negotiation failed: client connection to 127.28.7.189:39853: connect: Connection refused (error 111)
18:51:11.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.489329 28801 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.189:39853
18:51:11.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.489362 28886 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.189:39853 every 8 connection(s)
18:51:11.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.489662 28801 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/data/info.pb
18:51:11.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.490267 28887 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:51:11.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.491746 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 28801
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.491820 28702 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-1/wal/instance
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.492137 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/wal
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/logs
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data/info.pb
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:51:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.188:39825
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.188
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38211/jwks.json with env {}
18:51:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.492111 28887 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.498478 28887 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.502774 28887 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.503315 28824 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.188:39825: connect: Connection refused (error 111)
18:51:11.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.503930 28887 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.7.188:39825: Network error: Client connection negotiation failed: client connection to 127.28.7.188:39825: connect: Connection refused (error 111)
18:51:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.553177 28887 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } attempt: 1
18:51:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.554131 28887 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.7.188:39825: Network error: Client connection negotiation failed: client connection to 127.28.7.188:39825: connect: Connection refused (error 111)
18:51:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.577905 28797 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } attempt: 3
18:51:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.579161 28890 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.579370 28890 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.579401 28890 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.581038 28890 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:51:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.581101 28890 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.581122 28890 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:51:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.581144 28890 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:51:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.581167 28890 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:51:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.581183 28890 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38211/jwks.json
18:51:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.582352 28797 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.582837 28797 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.7.188:39825: Network error: Client connection negotiation failed: client connection to 127.28.7.188:39825: connect: Connection refused (error 111)
18:51:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.583362 28890 master_runner.cc:386] Master server non-default flags:
18:51:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:11.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/wal
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.188:39825
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38211/jwks.json
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data/info.pb
18:51:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.188
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.28890
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/logs
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.583599 28890 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.583971 28890 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.589707 28900 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.589771 28901 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.589720 28903 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.589880 28890 server_base.cc:1034] running on GCE node
18:51:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.590163 28890 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.590413 28890 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.591595 28890 hybrid_clock.cc:648] HybridClock initialized: now 1786474271591554 us; error 49 us; skew 500 ppm
18:51:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.593202 28890 webserver.cc:460] Webserver started at http://127.28.7.188:43917/ using document root <none> and password file <none>
18:51:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.593420 28890 fs_manager.cc:359] Metadata directory not provided
18:51:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.593459 28890 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.593559 28890 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:51:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.594552 28890 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data/instance:
18:51:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "020797b45b854a7db9dc738315ecada8"
18:51:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.595036 28890 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/wal/instance:
18:51:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "020797b45b854a7db9dc738315ecada8"
18:51:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.597055 28890 fs_manager.cc:691] Time spent creating directory manager: real 0.002s	user 0.000s	sys 0.003s
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.598019 28911 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.598204 28890 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.598299 28890 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/wal
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "020797b45b854a7db9dc738315ecada8"
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:11 on dist-test-slave-77v9"
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.598371 28890 fs_report.cc:383] FS layout report
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/wal
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/wal
18:51:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data/data
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.611624 28890 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.612031 28890 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.612190 28890 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.614015 28797 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } attempt: 1
18:51:11.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.614893 28797 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.7.188:39825: Network error: Client connection negotiation failed: client connection to 127.28.7.188:39825: connect: Connection refused (error 111)
18:51:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.616753 28890 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.188:39825
18:51:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.616786 28970 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.188:39825 every 8 connection(s)
18:51:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.617264 28890 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/data/info.pb
18:51:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.618072 28971 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:51:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.618511 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 28890
18:51:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.618628 28702 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/master-2/wal/instance
18:51:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.620059 28971 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.622400 28887 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } attempt: 2
18:51:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.627332 28971 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.629660 28887 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: Bootstrap starting.
18:51:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.630529 28887 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: Neither blocks nor log segments found. Creating new log.
18:51:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.630897 28887 log.cc:824] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: Log is configured to *not* fsync() on all Append() calls
18:51:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.631975 28887 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: No bootstrap required, opened a new log
18:51:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.634243 28887 raft_consensus.cc:348] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.634366 28887 raft_consensus.cc:374] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:51:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.634418 28887 raft_consensus.cc:729] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4b6180a702942fd9805fb8c3048953a, State: Initialized, Role: FOLLOWER
18:51:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.634545 28887 consensus_queue.cc:260] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.634872 28887 sys_catalog.cc:564] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [sys.catalog]: configured and running, proceeding with master startup.
18:51:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.634829 28983 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.634927 28983 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [sys.catalog]: This master's current role is: FOLLOWER
18:51:11.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.636861 28998 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:51:11.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.636934 28998 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:51:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.641450 28971 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } has no permanent_uuid. Determining permanent_uuid...
18:51:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.641297 28801 master_runner.cc:186] Error getting master registration for 127.28.7.190:38569: OK, instance_id { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" instance_seqno: 1786474271354712 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:51:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.644301 28801 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:51:11.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.646473 28971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: Bootstrap starting.
18:51:11.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.647346 28971 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: Neither blocks nor log segments found. Creating new log.
18:51:11.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.647686 28971 log.cc:824] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: Log is configured to *not* fsync() on all Append() calls
18:51:11.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.648581 28971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: No bootstrap required, opened a new log
18:51:11.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.651170 28971 raft_consensus.cc:348] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.651333 28971 raft_consensus.cc:374] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:51:11.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.651368 28971 raft_consensus.cc:729] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020797b45b854a7db9dc738315ecada8, State: Initialized, Role: FOLLOWER
18:51:11.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.651496 28971 consensus_queue.cc:260] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.651830 28971 sys_catalog.cc:564] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [sys.catalog]: configured and running, proceeding with master startup.
18:51:11.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.651801 29001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.651901 29001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [sys.catalog]: This master's current role is: FOLLOWER
18:51:11.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.654882 28890 master_runner.cc:186] Error getting master registration for 127.28.7.190:38569: OK, instance_id { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" instance_seqno: 1786474271354712 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:51:11.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.657218 28890 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:51:11.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.657738 29013 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:51:11.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.657815 29013 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:51:11.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.690083 28797 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } attempt: 2
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.696189 28797 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce: Bootstrap starting.
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.697053 28797 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce: Neither blocks nor log segments found. Creating new log.
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.697664 28797 log.cc:824] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce: Log is configured to *not* fsync() on all Append() calls
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.698710 28797 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce: No bootstrap required, opened a new log
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.700742 28797 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.700906 28797 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.700945 28797 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3487e6c7dce54f27bd27a4efdb1947ce, State: Initialized, Role: FOLLOWER
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.701086 28797 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.701442 28797 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: configured and running, proceeding with master startup.
18:51:11.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.702373 29017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.702723 29017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: This master's current role is: FOLLOWER
18:51:11.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.703291 29031 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:51:11.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.703625 29031 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:51:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.705513 28715 master_runner.cc:263] No leader master found from master 3487e6c7dce54f27bd27a4efdb1947ce
18:51:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.705559 28715 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:51:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.735414 28826 tablet.cc:2369] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:51:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.852538 28740 tablet.cc:2369] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:51:11.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:11.862979 28918 tablet.cc:2369] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:51:11.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.889220 29017 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:51:11.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.889348 29017 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.889616 29017 leader_election.cc:288] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d4b6180a702942fd9805fb8c3048953a (127.28.7.189:39853), 020797b45b854a7db9dc738315ecada8 (127.28.7.188:39825)
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.889896 28854 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d4b6180a702942fd9805fb8c3048953a" is_pre_election: true
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.889938 28943 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020797b45b854a7db9dc738315ecada8" is_pre_election: true
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.890094 28854 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3487e6c7dce54f27bd27a4efdb1947ce in term 0.
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.890096 28943 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3487e6c7dce54f27bd27a4efdb1947ce in term 0.
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.890302 28738 leader_election.cc:302] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 020797b45b854a7db9dc738315ecada8, 3487e6c7dce54f27bd27a4efdb1947ce; no voters: 
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.890479 29017 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Leader pre-election won for term 1
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.890549 29017 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:51:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.890565 29017 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 0 FOLLOWER]: Advancing to term 1
18:51:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.891198 29017 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.891302 29017 leader_election.cc:288] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [CANDIDATE]: Term 1 election: Requested vote from peers d4b6180a702942fd9805fb8c3048953a (127.28.7.189:39853), 020797b45b854a7db9dc738315ecada8 (127.28.7.188:39825)
18:51:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.891561 28943 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020797b45b854a7db9dc738315ecada8"
18:51:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.891577 28854 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d4b6180a702942fd9805fb8c3048953a"
18:51:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.891637 28943 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [term 0 FOLLOWER]: Advancing to term 1
18:51:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.891669 28854 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [term 0 FOLLOWER]: Advancing to term 1
18:51:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.892254 28943 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3487e6c7dce54f27bd27a4efdb1947ce in term 1.
18:51:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.892459 28738 leader_election.cc:302] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 020797b45b854a7db9dc738315ecada8, 3487e6c7dce54f27bd27a4efdb1947ce; no voters: 
18:51:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.892527 28854 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3487e6c7dce54f27bd27a4efdb1947ce in term 1.
18:51:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.892557 29017 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 1 FOLLOWER]: Leader election won for term 1
18:51:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.892693 29017 raft_consensus.cc:686] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [term 1 LEADER]: Becoming Leader. State: Replica: 3487e6c7dce54f27bd27a4efdb1947ce, State: Running, Role: LEADER
18:51:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.892783 29017 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } }
18:51:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.893304 29033 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3487e6c7dce54f27bd27a4efdb1947ce. Latest consensus state: current_term: 1 leader_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.893402 29033 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: This master's current role is: LEADER
18:51:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.893630 29035 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
18:51:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.893850 29035 catalog_manager.cc:1484] Initializing Kudu cluster ID...
18:51:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.895412 28943 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [term 1 FOLLOWER]: Refusing update from remote peer 3487e6c7dce54f27bd27a4efdb1947ce: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:51:11.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.895694 28854 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [term 1 FOLLOWER]: Refusing update from remote peer 3487e6c7dce54f27bd27a4efdb1947ce: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:51:11.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.895982 29017 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:51:11.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.896092 29033 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [LEADER]: Connected to new peer: Peer: permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:51:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.897470 28983 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3487e6c7dce54f27bd27a4efdb1947ce. Latest consensus state: current_term: 1 leader_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.897578 28983 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [sys.catalog]: This master's current role is: FOLLOWER
18:51:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.897861 29001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3487e6c7dce54f27bd27a4efdb1947ce. Latest consensus state: current_term: 1 leader_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.897972 29001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [sys.catalog]: This master's current role is: FOLLOWER
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.898247 29033 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.898324 29033 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: This master's current role is: LEADER
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.898380 29036 mvcc.cc:205] Tried to move back new op lower bound from 7317398617681022976 to 7317398617673613312. Current Snapshot: MvccSnapshot[applied={T|T < 7317398617681022976}]
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.898535 28983 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.898607 28983 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a [sys.catalog]: This master's current role is: FOLLOWER
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.898895 29017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.898972 29017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce [sys.catalog]: This master's current role is: LEADER
18:51:11.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.899101 29035 catalog_manager.cc:1347] Generated new cluster ID: 79a334b38de242299c58a1f9b4ceba9c
18:51:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.899148 29035 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
18:51:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.899523 29001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3487e6c7dce54f27bd27a4efdb1947ce" member_type: VOTER last_known_addr { host: "127.28.7.190" port: 38569 } } peers { permanent_uuid: "d4b6180a702942fd9805fb8c3048953a" member_type: VOTER last_known_addr { host: "127.28.7.189" port: 39853 } } peers { permanent_uuid: "020797b45b854a7db9dc738315ecada8" member_type: VOTER last_known_addr { host: "127.28.7.188" port: 39825 } } }
18:51:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.899600 29001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8 [sys.catalog]: This master's current role is: FOLLOWER
18:51:11.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.899655 29039 mvcc.cc:205] Tried to move back new op lower bound from 7317398617681022976 to 7317398617673613312. Current Snapshot: MvccSnapshot[applied={T|T < 7317398617681022976}]
18:51:11.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.901260 29037 mvcc.cc:205] Tried to move back new op lower bound from 7317398617681022976 to 7317398617673613312. Current Snapshot: MvccSnapshot[applied={T|T < 7317398617681022976}]
18:51:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.914604 29035 catalog_manager.cc:1370] Generated new certificate authority record
18:51:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.915256 29035 catalog_manager.cc:1504] Loading token signing keys...
18:51:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.919577 29035 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 3487e6c7dce54f27bd27a4efdb1947ce: Generated new TSK 0
18:51:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.920040 29035 catalog_manager.cc:1514] Initializing in-progress tserver states...
18:51:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:11.931372 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:0
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.018407 29045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.018698 29045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.018735 29045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.020226 29045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.020313 29045 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.129
18:51:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.021795 29045 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:0
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29045
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.022073 29045 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.022437 29045 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.028215 29050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.028221 29054 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.028215 29052 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.028431 29045 server_base.cc:1034] running on GCE node
18:51:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.028630 29045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.028947 29045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:12.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.030125 29045 hybrid_clock.cc:648] HybridClock initialized: now 1786474272030116 us; error 46 us; skew 500 ppm
18:51:12.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.031641 29045 webserver.cc:460] Webserver started at http://127.28.7.129:37913/ using document root <none> and password file <none>
18:51:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.031955 29045 fs_manager.cc:359] Metadata directory not provided
18:51:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.032038 29045 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.032186 29045 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:51:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.033293 29045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/instance:
18:51:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.033697 29045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal/instance:
18:51:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.035274 29045 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.002s	sys 0.000s
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.036175 29061 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.036410 29045 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.036504 29045 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.036589 29045 fs_report.cc:383] FS layout report
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/data
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.062345 29045 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.062745 29045 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.062911 29045 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.063206 29045 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.063655 29045 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:51:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.063719 29045 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:51:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.063755 29045 ts_tablet_manager.cc:610] Registered 0 tablets
18:51:12.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.063771 29045 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:51:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.070182 29045 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.129:43067
18:51:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.070230 29189 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.129:43067 every 8 connection(s)
18:51:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.070623 29045 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077225 29190 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077226 29192 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077245 29193 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077375 29190 heartbeater.cc:464] Registering TS with master...
18:51:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077375 29192 heartbeater.cc:464] Registering TS with master...
18:51:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077380 29193 heartbeater.cc:464] Registering TS with master...
18:51:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077594 29192 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077594 29190 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.077594 29193 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.078133 28753 ts_manager.cc:194] Registered new tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.078226 28842 ts_manager.cc:194] Registered new tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.078406 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 29045
18:51:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.078474 28702 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal/instance
18:51:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.078910 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.129:45299
18:51:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.079758 28923 ts_manager.cc:194] Registered new tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:12.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.079896 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:0
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:12.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.170244 29198 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:12.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.170449 29198 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:12.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.170468 29198 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:12.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.171517 29198 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:12.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.171566 29198 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.130
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.172894 29198 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:0
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29198
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.173089 29198 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.173384 29198 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.178515 29207 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.178515 29205 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.178540 29209 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.178628 29198 server_base.cc:1034] running on GCE node
18:51:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.179216 29198 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.179417 29198 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.180581 29198 hybrid_clock.cc:648] HybridClock initialized: now 1786474272180561 us; error 37 us; skew 500 ppm
18:51:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.182025 29198 webserver.cc:460] Webserver started at http://127.28.7.130:45681/ using document root <none> and password file <none>
18:51:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.182336 29198 fs_manager.cc:359] Metadata directory not provided
18:51:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.182420 29198 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.182561 29198 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:51:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.183559 29198 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/instance:
18:51:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.183976 29198 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal/instance:
18:51:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.185531 29198 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.000s	sys 0.003s
18:51:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.186420 29217 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.186615 29198 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:51:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.186717 29198 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.186877 29198 fs_report.cc:383] FS layout report
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/data
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.197201 29198 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.197451 29198 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.197566 29198 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.197737 29198 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.198006 29198 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:51:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.198040 29198 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:51:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.198061 29198 ts_tablet_manager.cc:610] Registered 0 tablets
18:51:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.198073 29198 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:51:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.204596 29198 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.130:46345
18:51:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.204684 29351 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.130:46345 every 8 connection(s)
18:51:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.204979 29198 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.206223 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 29198
18:51:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.206317 28702 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal/instance
18:51:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.207535 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:0
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.211936 29354 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.212036 29354 heartbeater.cc:464] Registering TS with master...
18:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.212138 29353 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.212196 29353 heartbeater.cc:464] Registering TS with master...
18:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.212252 29354 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.212262 29353 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.212647 28751 ts_manager.cc:194] Registered new tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.213140 28842 ts_manager.cc:194] Registered new tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.213197 28751 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.130:34365
18:51:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.213660 29352 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.213738 29352 heartbeater.cc:464] Registering TS with master...
18:51:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.213814 29352 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:12.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.214110 28929 ts_manager.cc:194] Registered new tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.293962 29360 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.294176 29360 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.294196 29360 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.295393 29360 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.295452 29360 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.131
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.296957 29360 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:0
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29360
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.297170 29360 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.297429 29360 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.302824 29368 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.302871 29366 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:12.302877 29365 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.303082 29360 server_base.cc:1034] running on GCE node
18:51:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.303503 29360 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.303774 29360 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:12.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.304929 29360 hybrid_clock.cc:648] HybridClock initialized: now 1786474272304909 us; error 38 us; skew 500 ppm
18:51:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.306180 29360 webserver.cc:460] Webserver started at http://127.28.7.131:33917/ using document root <none> and password file <none>
18:51:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.306406 29360 fs_manager.cc:359] Metadata directory not provided
18:51:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.306463 29360 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.306557 29360 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:51:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.307504 29360 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/instance:
18:51:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.307847 29360 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal/instance:
18:51:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.309448 29360 fs_manager.cc:691] Time spent creating directory manager: real 0.001s	user 0.003s	sys 0.000s
18:51:12.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.310338 29375 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:12.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.310497 29360 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:12.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.310581 29360 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.310659 29360 fs_report.cc:383] FS layout report
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/data
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.330479 29360 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.330891 29360 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.331038 29360 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.331259 29360 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.331694 29360 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:51:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.331758 29360 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:51:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.331800 29360 ts_tablet_manager.cc:610] Registered 0 tablets
18:51:12.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.331815 29360 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:51:12.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.338479 29360 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.131:33291
18:51:12.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.338548 29508 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.131:33291 every 8 connection(s)
18:51:12.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.338822 29360 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345521 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 29360
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345541 29513 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345638 28702 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal/instance
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345577 29514 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345552 29509 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345651 29513 heartbeater.cc:464] Registering TS with master...
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345685 29514 heartbeater.cc:464] Registering TS with master...
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345713 29509 heartbeater.cc:464] Registering TS with master...
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345908 29513 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345908 29509 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.345908 29514 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.346371 28751 ts_manager.cc:194] Registered new tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.346441 28929 ts_manager.cc:194] Registered new tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.346438 28842 ts_manager.cc:194] Registered new tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:12.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.346879 28751 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.131:56965
18:51:12.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.349201 28702 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
18:51:12.350 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:51:12.351 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

18:51:12.362 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.28.7.190"
      port: 38569
    }
    bound_http_address {
      host: "127.28.7.190"
      port: 40011
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.28.7.189"
      port: 39853
    }
    bound_http_address {
      host: "127.28.7.189"
      port: 39935
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.28.7.188"
      port: 39825
    }
    bound_http_address {
      host: "127.28.7.188"
      port: 43917
    }
  }
}

18:51:12.367 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

18:51:12.369 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.28.7.129"
      port: 43067
    }
    bound_http_address {
      host: "127.28.7.129"
      port: 37913
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.28.7.130"
      port: 46345
    }
    bound_http_address {
      host: "127.28.7.130"
      port: 45681
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.28.7.131"
      port: 33291
    }
    bound_http_address {
      host: "127.28.7.131"
      port: 33917
    }
  }
}

18:51:12.369 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
18:51:12.553 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
18:51:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.637463 28998 catalog_manager.cc:1259] Loaded cluster ID: 79a334b38de242299c58a1f9b4ceba9c
18:51:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.637566 28998 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: loading cluster ID for follower catalog manager: success
18:51:12.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.638976 28998 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: acquiring CA information for follower catalog manager: success
18:51:12.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.639405 28998 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d4b6180a702942fd9805fb8c3048953a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:51:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.658329 29013 catalog_manager.cc:1259] Loaded cluster ID: 79a334b38de242299c58a1f9b4ceba9c
18:51:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.658408 29013 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: loading cluster ID for follower catalog manager: success
18:51:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.659453 29013 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: acquiring CA information for follower catalog manager: success
18:51:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:12.659700 29013 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 020797b45b854a7db9dc738315ecada8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:51:12.733 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.7.189:39853(127.28.7.189:39853)] connecting to peer
18:51:12.790 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.7.188:39825(127.28.7.188:39825)] connecting to peer
18:51:12.791 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.7.190:38569(127.28.7.190:38569)] connecting to peer
18:51:12.812 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.7.188:39825(127.28.7.188:39825)] Successfully connected to peer
18:51:12.812 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.7.189:39853(127.28.7.189:39853)] Successfully connected to peer
18:51:12.878 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.7.190:38569(127.28.7.190:38569)] Successfully connected to peer
18:51:12.898 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.28.7.188
18:51:12.898 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.28.7.189
18:51:12.999 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.28.7.190
18:51:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.079871 29192 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:13.120 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc65df7fe, L:/127.0.0.1:34604 - R:/127.28.7.189:39853] using SASL/PLAIN
18:51:13.120 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x608fd419, L:/127.0.0.1:45120 - R:/127.28.7.188:39825] using SASL/PLAIN
18:51:13.135 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf9972725, L:/127.0.0.1:41796 - R:/127.28.7.190:38569] using SASL/PLAIN
18:51:13.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
18:51:13.173 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=481), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.7.189:39853, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.7.188:39825, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.7.190:38569, [438ms] Sub RPC ConnectToMaster: received response from server master-127.28.7.188:39825: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.28.7.189:39853: OK, [446ms] Sub RPC ConnectToMaster: received response from server master-127.28.7.190:38569: OK, deferred=null) after lookup
18:51:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.214361 29353 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.236493 28751 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41796:
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680"
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:51:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
18:51:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:51:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:51:13.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:13.238245 28751 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
18:51:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.247939 29274 tablet_service.cc:1467] Processing CreateTablet for tablet 1a1e672af9074011b1eba5773e6d0f4a (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680 [id=b4df742abb2344159715023c20136e6a]), partition=RANGE (key) PARTITION UNBOUNDED
18:51:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.247978 29430 tablet_service.cc:1467] Processing CreateTablet for tablet 1a1e672af9074011b1eba5773e6d0f4a (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680 [id=b4df742abb2344159715023c20136e6a]), partition=RANGE (key) PARTITION UNBOUNDED
18:51:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.248153 29113 tablet_service.cc:1467] Processing CreateTablet for tablet 1a1e672af9074011b1eba5773e6d0f4a (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680 [id=b4df742abb2344159715023c20136e6a]), partition=RANGE (key) PARTITION UNBOUNDED
18:51:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.248338 29430 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a1e672af9074011b1eba5773e6d0f4a. 1 dirs total, 0 dirs full, 0 dirs failed
18:51:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.248337 29274 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a1e672af9074011b1eba5773e6d0f4a. 1 dirs total, 0 dirs full, 0 dirs failed
18:51:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.248461 29113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a1e672af9074011b1eba5773e6d0f4a. 1 dirs total, 0 dirs full, 0 dirs failed
18:51:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.251444 29545 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap starting.
18:51:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.252115 29545 tablet_bootstrap.cc:654] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Neither blocks nor log segments found. Creating new log.
18:51:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.252468 29545 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Log is configured to *not* fsync() on all Append() calls
18:51:13.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.252775 29544 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap starting.
18:51:13.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.252979 29543 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap starting.
18:51:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.253458 29544 tablet_bootstrap.cc:654] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Neither blocks nor log segments found. Creating new log.
18:51:13.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.253684 29543 tablet_bootstrap.cc:654] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Neither blocks nor log segments found. Creating new log.
18:51:13.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.253750 29544 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Log is configured to *not* fsync() on all Append() calls
18:51:13.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.253971 29543 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Log is configured to *not* fsync() on all Append() calls
18:51:13.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.254391 29545 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: No bootstrap required, opened a new log
18:51:13.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.254451 29545 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent bootstrapping tablet: real 0.003s	user 0.001s	sys 0.000s
18:51:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.255990 29544 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: No bootstrap required, opened a new log
18:51:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.256130 29543 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: No bootstrap required, opened a new log
18:51:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.256191 29543 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent bootstrapping tablet: real 0.003s	user 0.001s	sys 0.000s
18:51:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.256393 29544 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent bootstrapping tablet: real 0.004s	user 0.002s	sys 0.000s
18:51:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.256935 29545 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.257267 29545 raft_consensus.cc:374] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:51:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.257449 29545 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Initialized, Role: FOLLOWER
18:51:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.257725 29545 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.258127 29545 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent starting tablet: real 0.004s	user 0.003s	sys 0.000s
18:51:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.258821 29543 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.259140 29543 raft_consensus.cc:374] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:51:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.259302 29543 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc2bbe61c8493aa2bc1bfa367b392b, State: Initialized, Role: FOLLOWER
18:51:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.259572 29543 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.259753 29544 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.259972 29543 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent starting tablet: real 0.004s	user 0.000s	sys 0.004s
18:51:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.260109 29544 raft_consensus.cc:374] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:51:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.260267 29544 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Initialized, Role: FOLLOWER
18:51:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.260574 29544 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.261340 29544 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent starting tablet: real 0.005s	user 0.003s	sys 0.000s
18:51:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.261845 29513 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:13.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table b4df742abb2344159715023c20136e6a
18:51:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:13.322549 29194 tablet.cc:2369] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:51:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:13.340090 29515 tablet.cc:2369] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:51:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.410033 29552 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:51:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.410178 29552 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.410542 29552 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415210 29136 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" is_pre_election: true
18:51:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415205 29298 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" is_pre_election: true
18:51:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415422 29298 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 0.
18:51:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415422 29136 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 0.
18:51:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415720 29380 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: 
18:51:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415895 29552 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Leader pre-election won for term 1
18:51:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415948 29552 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:51:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.415971 29552 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 0 FOLLOWER]: Advancing to term 1
18:51:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.416638 29552 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.416788 29552 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 1 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.416993 29136 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.417060 29298 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.417098 29136 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 0 FOLLOWER]: Advancing to term 1
18:51:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.417135 29298 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 0 FOLLOWER]: Advancing to term 1
18:51:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.417971 29298 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 1.
18:51:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.417971 29136 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 1.
18:51:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.418193 29380 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: 
18:51:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.418298 29552 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 1 FOLLOWER]: Leader election won for term 1
18:51:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.418453 29552 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 1 LEADER]: Becoming Leader. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Running, Role: LEADER
18:51:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.418526 29552 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.419744 28751 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 reported cstate change: term changed from 0 to 1, leader changed from <none> to f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131). New cstate: current_term: 1 leader_uuid: "f8e2085855944ef6829cc9ccffc14be4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: UNKNOWN } } }
18:51:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:13.456429 29355 tablet.cc:2369] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:51:13.457 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table b4df742abb2344159715023c20136e6a
18:51:13.477 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:13.487 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299989, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:13.490 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [8ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=null) after lookup
18:51:13.501 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:13.503 [DEBUG - kudu-nio-1] (Connection.java:757) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] Successfully connected to peer
18:51:13.555 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8a60e83d, L:/127.0.0.1:46016 - R:/127.28.7.131:33291] using TOKEN/null
18:51:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.558004 29298 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 1 FOLLOWER]: Refusing update from remote peer f8e2085855944ef6829cc9ccffc14be4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:51:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.558043 29136 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 1 FOLLOWER]: Refusing update from remote peer f8e2085855944ef6829cc9ccffc14be4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:51:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.558425 29552 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:51:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.558593 29556 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:51:13.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.561527 29559 mvcc.cc:205] Tried to move back new op lower bound from 7317398624490610688 to 7317398623922958336. Current Snapshot: MvccSnapshot[applied={T|T < 7317398624490610688}]
18:51:13.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.563205 29561 mvcc.cc:205] Tried to move back new op lower bound from 7317398624490610688 to 7317398623922958336. Current Snapshot: MvccSnapshot[applied={T|T < 7317398624490610688 or (T in {7317398624490610688})}]
18:51:13.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.563674 29560 mvcc.cc:205] Tried to move back new op lower bound from 7317398624490610688 to 7317398623922958336. Current Snapshot: MvccSnapshot[applied={T|T < 7317398624490610688 or (T in {7317398624490610688})}]
18:51:13.641 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:13.643 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.131:33291
18:51:13.645 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:13.645762 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 29360
18:51:13.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:13.660 [DEBUG - kudu-nio-1] (Connection.java:239) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] handling channelInactive
18:51:13.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state READY due to: connection closed
18:51:13.661 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: connection closed
18:51:13.675 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:13.676 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:13.676 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1894505982(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@6248d890 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11d2b2c7 -> wakeup thread Time-limited test)) after lookup
18:51:13.677 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:13.679 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.679 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.695 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:13.695 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:13.696 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [20ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [22ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [22ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1894505982(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@6248d890 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11d2b2c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348))) after lookup
18:51:13.697 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:13.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.698 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.716 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:13.717 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:13.718 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [20ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [22ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [22ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [41ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [59ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1894505982(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@6248d890 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11d2b2c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966))) after lookup
18:51:13.719 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:13.720 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.720 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.720 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.856 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:13.856 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:13.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=199), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [20ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [22ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [22ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [41ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [59ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [61ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [63ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [63ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [196ms] refreshing cache from master, [196ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [198ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1894505982(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@6248d890 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966) -> (continuation of Deferred@754113561 after retry RPC@1953686600), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11d2b2c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966) -> (continuation of Deferred@754113561 after retry RPC@1953686600))) after lookup
18:51:13.858 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:13.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.860 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.956 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:13.956 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:13.957 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=299), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [20ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [22ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [22ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [41ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [59ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [61ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [63ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [63ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [196ms] refreshing cache from master, [196ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [198ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [201ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [203ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [203ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [296ms] refreshing cache from master, [296ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [299ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1894505982(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@6248d890 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966) -> (continuation of Deferred@754113561 after retry RPC@1953686600) -> (continuation of Deferred@1330954694 after retry RPC@744963264), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11d2b2c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966) -> (continuation of Deferred@754113561 after retry RPC@1953686600) -> (continuation of Deferred@1330954694 after retry RPC@744963264))) after lookup
18:51:13.958 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:13.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:13.959 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:15.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.094237 29573 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:15.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.094343 29575 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:15.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.094422 29575 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.094408 29573 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.094745 29575 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.095116 29573 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.095199 29218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.095288 29062 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:15.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.095686 29218 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:15.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.096487 29062 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.098891 29298 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" is_pre_election: true
18:51:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.098989 29298 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 1.
18:51:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.099172 29065 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.099393 29573 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 1 FOLLOWER]: Leader pre-election won for term 2
18:51:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.099355 29136 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" is_pre_election: true
18:51:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.099458 29573 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 1 FOLLOWER]: Starting leader election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.099484 29573 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 1 FOLLOWER]: Advancing to term 2
18:51:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.100229 29573 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.100389 29136 raft_consensus.cc:2379] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fafc2bbe61c8493aa2bc1bfa367b392b in current term 2: Already voted for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in this term.
18:51:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.100607 29573 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 2 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.100672 29298 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.100759 29298 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 1 FOLLOWER]: Advancing to term 2
18:51:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.100870 29220 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4
18:51:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.101166 29062 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.101637 29298 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 2.
18:51:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.101794 29575 raft_consensus.cc:2726] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
18:51:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.101857 29065 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.101962 29573 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Leader election won for term 2
18:51:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.102121 29573 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 LEADER]: Becoming Leader. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Running, Role: LEADER
18:51:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.102236 29573 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.103036 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 reported cstate change: term changed from 1 to 2, leader changed from f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131) to 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129). New cstate: current_term: 2 leader_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: HEALTHY } } }
18:51:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.482195 29062 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 -> Peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Couldn't send request to peer f8e2085855944ef6829cc9ccffc14be4. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:15.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.559909 29298 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Refusing update from remote peer 2dc38ce1e08949da90122cd3ab2b4a31: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
18:51:15.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.560385 29573 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
18:51:15.675 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:15.676 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:15.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2018), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [20ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [22ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [22ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [41ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [59ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [61ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [63ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [63ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [196ms] refreshing cache from master, [196ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [198ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [201ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [203ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [203ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [296ms] refreshing cache from master, [296ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [299ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [301ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [302ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [302ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [2015ms] refreshing cache from master, [2015ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [2018ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1894505982(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@6248d890 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966) -> (continuation of Deferred@754113561 after retry RPC@1953686600) -> (continuation of Deferred@1330954694 after retry RPC@744963264) -> (continuation of Deferred@88643358 after retry RPC@164151623), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@11d2b2c7 -> wakeup thread Time-limited test -> (continuation of Deferred@1748609360 after retry RPC@2009281348) -> (continuation of Deferred@1353316804 after retry RPC@1042200966) -> (continuation of Deferred@754113561 after retry RPC@1953686600) -> (continuation of Deferred@1330954694 after retry RPC@744963264) -> (continuation of Deferred@88643358 after retry RPC@164151623))) after lookup
18:51:15.678 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:15.680 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] Successfully connected to peer
18:51:15.706 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2d47decc, L:/127.0.0.1:44384 - R:/127.28.7.129:43067] using TOKEN/null
18:51:15.720 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.131:33291
18:51:15.722 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.722569 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:15.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.815626 29613 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.815869 29613 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.815894 29613 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.817687 29613 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.817781 29613 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.131
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.820007 29613 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29613
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.820333 29613 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.820724 29613 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.826756 29621 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.827019 29613 server_base.cc:1034] running on GCE node
18:51:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.827114 29620 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:15.826772 29624 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.827597 29613 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:15.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.827903 29613 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:15.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.829073 29613 hybrid_clock.cc:648] HybridClock initialized: now 1786474275829054 us; error 40 us; skew 500 ppm
18:51:15.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.830566 29613 webserver.cc:460] Webserver started at http://127.28.7.131:33917/ using document root <none> and password file <none>
18:51:15.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.830796 29613 fs_manager.cc:359] Metadata directory not provided
18:51:15.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.830850 29613 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.832381 29613 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:51:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.833199 29631 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.833395 29613 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.833468 29613 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.833823 29613 fs_report.cc:383] FS layout report
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/data
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:15.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.854426 29613 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:15.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.854751 29613 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:15.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.854863 29613 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:15.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.855084 29613 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:15.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.855626 29639 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:15.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.857107 29613 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:15.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.857161 29613 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:15.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.857190 29613 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.858040 29613 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.858094 29613 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.858223 29639 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap starting.
18:51:15.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.866398 29613 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.131:33291
18:51:15.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.866588 29773 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.131:33291 every 8 connection(s)
18:51:15.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.867219 29613 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.869050 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 29613
18:51:15.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.869916 29639 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Log is configured to *not* fsync() on all Append() calls
18:51:15.874 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:15.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.875114 29639 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:15.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.875656 29639 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap complete.
18:51:15.877 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.129:43067
18:51:15.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.877179 29639 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent bootstrapping tablet: real 0.019s	user 0.004s	sys 0.011s
18:51:15.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:51:15.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.878005 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 29045
18:51:15.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.883103 29777 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:15.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.883244 29777 heartbeater.cc:464] Registering TS with master...
18:51:15.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.883471 29777 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:15.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.884541 28842 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:15.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.884661 29775 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.884734 29775 heartbeater.cc:464] Registering TS with master...
18:51:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.884850 29775 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.884672 29639 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.885505 29639 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Initialized, Role: FOLLOWER
18:51:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.885686 29639 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.886394 29639 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent starting tablet: real 0.009s	user 0.002s	sys 0.000s
18:51:15.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.887123 28753 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:15.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.887180 29774 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:15.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.887221 29774 heartbeater.cc:464] Registering TS with master...
18:51:15.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.887296 29774 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:15.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.887635 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.131:41645
18:51:15.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.888285 28923 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:15.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:15.888389 29775 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:15.890 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] handling channelInactive
18:51:15.890 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:15.890 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state READY due to: connection closed
18:51:15.891 [DEBUG - Time-limited test] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:15.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.894 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.916 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:15.916 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:15.917 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=25), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [24ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1908249656(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@199a9176 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3cc35a90 -> wakeup thread Time-limited test)) after lookup
18:51:15.918 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:15.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.920 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.935 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:15.936 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:15.937 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=45), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [24ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [26ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [29ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1908249656(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@199a9176 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3cc35a90 -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132))) after lookup
18:51:15.937 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:15.939 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.939 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.939 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.957 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:15.958 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:15.959 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=67), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [24ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [26ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [29ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [46ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [48ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [48ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [65ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1908249656(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@199a9176 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3cc35a90 -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753))) after lookup
18:51:15.960 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:15.962 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.962 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:15.962 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:16.036 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:16.037 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:16.038 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=146), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [24ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [26ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [29ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [46ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [48ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [48ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [65ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [68ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [71ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [72ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [144ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1908249656(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@199a9176 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3cc35a90 -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667))) after lookup
18:51:16.039 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:16.040 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:16.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:16.041 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:16.457 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:16.457 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:16.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=566), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [24ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [26ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [29ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [46ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [48ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [48ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [65ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [68ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [71ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [72ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [144ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [148ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [150ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [150ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [562ms] refreshing cache from master, [562ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [564ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1908249656(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@199a9176 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667) -> (continuation of Deferred@322817566 after retry RPC@1597173067), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3cc35a90 -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667) -> (continuation of Deferred@322817566 after retry RPC@1597173067))) after lookup
18:51:16.459 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:16.462 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:16.462 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:16.463 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:17.036 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:17.037 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:17.037 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1146), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [24ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [26ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [29ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [46ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [48ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [48ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [65ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [68ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [71ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [72ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [144ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [148ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [150ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [150ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [562ms] refreshing cache from master, [562ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [564ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [568ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [572ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [572ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [1141ms] refreshing cache from master, [1142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [1144ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1908249656(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@199a9176 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667) -> (continuation of Deferred@322817566 after retry RPC@1597173067) -> (continuation of Deferred@813772441 after retry RPC@158229983), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3cc35a90 -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667) -> (continuation of Deferred@322817566 after retry RPC@1597173067) -> (continuation of Deferred@813772441 after retry RPC@158229983))) after lookup
18:51:17.038 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:17.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:17.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:17.040 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.218896 29788 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 2dc38ce1e08949da90122cd3ab2b4a31)
18:51:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.219046 29788 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.219286 29788 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:17.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.220335 29220 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.223412 29720 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "f8e2085855944ef6829cc9ccffc14be4" is_pre_election: true
18:51:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.223582 29720 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc2bbe61c8493aa2bc1bfa367b392b in term 1.
18:51:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.223819 29218 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: f8e2085855944ef6829cc9ccffc14be4, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31
18:51:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.224004 29788 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Leader pre-election won for term 3
18:51:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.224049 29788 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Starting leader election (detected failure of leader 2dc38ce1e08949da90122cd3ab2b4a31)
18:51:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.224066 29788 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 2 FOLLOWER]: Advancing to term 3
18:51:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.224872 29788 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.225054 29788 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 3 election: Requested vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.225255 29720 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.225389 29720 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 1 FOLLOWER]: Advancing to term 3
18:51:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.225703 29220 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:17.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.226696 29720 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc2bbe61c8493aa2bc1bfa367b392b in term 3.
18:51:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.226928 29218 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: f8e2085855944ef6829cc9ccffc14be4, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31
18:51:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.227039 29788 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Leader election won for term 3
18:51:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.227192 29788 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 LEADER]: Becoming Leader. State: Replica: fafc2bbe61c8493aa2bc1bfa367b392b, State: Running, Role: LEADER
18:51:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.227303 29788 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.228340 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b reported cstate change: term changed from 2 to 3, leader changed from 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129) to fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130). New cstate: current_term: 3 leader_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: UNKNOWN } } }
18:51:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.613598 29720 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 3 FOLLOWER]: Refusing update from remote peer fafc2bbe61c8493aa2bc1bfa367b392b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
18:51:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.614109 29788 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
18:51:17.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.617264 29220 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b -> Peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Couldn't send request to peer 2dc38ce1e08949da90122cd3ab2b4a31. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:17.676 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:17.677 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:17.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1786), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [24ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [26ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [29ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [46ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [48ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [48ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [62ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [65ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [68ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [71ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [72ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [144ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [148ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [150ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [150ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [562ms] refreshing cache from master, [562ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [564ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [568ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [572ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [572ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [1141ms] refreshing cache from master, [1142ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [1144ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [1147ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [1149ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [1149ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [1782ms] refreshing cache from master, [1782ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [1784ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1908249656(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@199a9176 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667) -> (continuation of Deferred@322817566 after retry RPC@1597173067) -> (continuation of Deferred@813772441 after retry RPC@158229983) -> (continuation of Deferred@1902033046 after retry RPC@1722404148), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3cc35a90 -> wakeup thread Time-limited test -> (continuation of Deferred@1867094695 after retry RPC@1232624132) -> (continuation of Deferred@1227916033 after retry RPC@61960753) -> (continuation of Deferred@1527552417 after retry RPC@1394886667) -> (continuation of Deferred@322817566 after retry RPC@1597173067) -> (continuation of Deferred@813772441 after retry RPC@158229983) -> (continuation of Deferred@1902033046 after retry RPC@1722404148))) after lookup
18:51:17.678 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:17.681 [DEBUG - kudu-nio-1] (Connection.java:757) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] Successfully connected to peer
18:51:17.704 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb9fe8a75, L:/127.0.0.1:51534 - R:/127.28.7.130:46345] using TOKEN/null
18:51:17.716 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.129:43067
18:51:17.717 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.717981 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:43067
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.805274 29821 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.805483 29821 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.805503 29821 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.806620 29821 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.806669 29821 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.129
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.807976 29821 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:43067
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.29821
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.808192 29821 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.808456 29821 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:17.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.813884 29831 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:17.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.813941 29827 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:17.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:17.813843 29828 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.814941 29821 server_base.cc:1034] running on GCE node
18:51:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.815208 29821 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.815536 29821 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.816708 29821 hybrid_clock.cc:648] HybridClock initialized: now 1786474277816687 us; error 49 us; skew 500 ppm
18:51:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.822899 29821 webserver.cc:460] Webserver started at http://127.28.7.129:37913/ using document root <none> and password file <none>
18:51:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.823261 29821 fs_manager.cc:359] Metadata directory not provided
18:51:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.823376 29821 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.824967 29821 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.002s	sys 0.000s
18:51:17.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.825841 29839 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:17.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.826103 29821 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:17.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.826226 29821 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:17.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:17.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.826769 29821 fs_report.cc:383] FS layout report
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/data
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.845358 29821 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.845772 29821 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.845966 29821 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.846248 29821 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.846801 29846 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.848131 29821 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.848204 29821 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.848243 29821 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.849045 29821 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.849107 29821 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.001s	sys 0.000s
18:51:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.849159 29846 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap starting.
18:51:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.856822 29821 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.129:43067
18:51:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.856935 29966 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.129:43067 every 8 connection(s)
18:51:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.857293 29821 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.857434 29846 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Log is configured to *not* fsync() on all Append() calls
18:51:17.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.860746 29846 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:17.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.861130 29846 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap complete.
18:51:17.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.862321 29846 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent bootstrapping tablet: real 0.013s	user 0.002s	sys 0.010s
18:51:17.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.863476 29846 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.863821 29846 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Initialized, Role: FOLLOWER
18:51:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.863996 29846 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.864229 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 29821
18:51:17.864 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.864315 29846 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent starting tablet: real 0.002s	user 0.002s	sys 0.001s
18:51:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.864992 29968 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.865541 29968 heartbeater.cc:464] Registering TS with master...
18:51:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.865739 29968 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.865768 29969 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.865819 29969 heartbeater.cc:464] Registering TS with master...
18:51:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.865885 29969 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.866361 28753 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.866881 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.129:46775
18:51:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.865021 29967 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.866988 29967 heartbeater.cc:464] Registering TS with master...
18:51:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.867065 29967 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.867184 28842 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.867980 29968 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.868383 28923 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:17.869 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:17.871 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.130:46345
18:51:17.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:51:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:17.871968 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 29198
18:51:17.881 [DEBUG - kudu-nio-1] (Connection.java:239) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] handling channelInactive
18:51:17.881 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:17.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state READY due to: connection closed
18:51:17.882 [DEBUG - Time-limited test] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:17.884 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.884 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.884 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.896 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:17.897 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:17.897 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [4ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1961228486(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@61191911 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d559c7 -> wakeup thread Time-limited test)) after lookup
18:51:17.899 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:17.901 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.901 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.916 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:17.916 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:17.917 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [4ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [17ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [20ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [21ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [34ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1961228486(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@61191911 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d559c7 -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605))) after lookup
18:51:17.917 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:17.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.918 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.956 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:17.957 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:17.957 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [4ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [17ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [20ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [21ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [34ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [36ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [37ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [37ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [74ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1961228486(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@61191911 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d559c7 -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625))) after lookup
18:51:17.958 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:17.960 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.960 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.961 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:17.996 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:17.997 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:17.997 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=115), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [4ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [17ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [20ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [21ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [34ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [36ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [37ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [37ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [74ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [77ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [79ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [80ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [114ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1961228486(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@61191911 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d559c7 -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184))) after lookup
18:51:17.998 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:17.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.000 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.000 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.376 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:18.377 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:18.378 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=496), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [4ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [17ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [20ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [21ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [34ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [36ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [37ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [37ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [74ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [77ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [79ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [80ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [114ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [117ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [119ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [119ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [492ms] refreshing cache from master, [492ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [495ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1961228486(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@61191911 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184) -> (continuation of Deferred@519349224 after retry RPC@1467392479), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d559c7 -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184) -> (continuation of Deferred@519349224 after retry RPC@1467392479))) after lookup
18:51:18.378 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:18.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.381 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.617 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:18.617 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:18.618 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=736), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [4ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [17ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [20ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [21ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [34ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [36ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [37ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [37ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [74ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [77ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [79ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [80ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [114ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [117ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [119ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [119ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [492ms] refreshing cache from master, [492ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [495ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [497ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [500ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [500ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [732ms] refreshing cache from master, [732ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [734ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1961228486(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@61191911 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184) -> (continuation of Deferred@519349224 after retry RPC@1467392479) -> (continuation of Deferred@1674320139 after retry RPC@22422772), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d559c7 -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184) -> (continuation of Deferred@519349224 after retry RPC@1467392479) -> (continuation of Deferred@1674320139 after retry RPC@22422772))) after lookup
18:51:18.620 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:18.621 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.621 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:18.621 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:19.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.201107 29984 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:51:19.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.201325 29984 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:19.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.201771 29984 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:19.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:19.202296 29843 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:19.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:19.202852 29843 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:19.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.205632 29720 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "f8e2085855944ef6829cc9ccffc14be4" is_pre_election: true
18:51:19.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.205972 29840 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31; no voters: f8e2085855944ef6829cc9ccffc14be4, fafc2bbe61c8493aa2bc1bfa367b392b
18:51:19.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.206148 29984 raft_consensus.cc:2726] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
18:51:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.215677 29991 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fafc2bbe61c8493aa2bc1bfa367b392b)
18:51:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.215770 29991 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.216123 29991 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:19.216601 29636 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:19.217334 29636 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.219960 29919 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" is_pre_election: true
18:51:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.220119 29919 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 2.
18:51:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.220400 29634 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: fafc2bbe61c8493aa2bc1bfa367b392b
18:51:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.220554 29991 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 3 FOLLOWER]: Leader pre-election won for term 4
18:51:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.220611 29991 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 3 FOLLOWER]: Starting leader election (detected failure of leader fafc2bbe61c8493aa2bc1bfa367b392b)
18:51:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.220628 29991 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 3 FOLLOWER]: Advancing to term 4
18:51:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.221356 29991 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.221516 29991 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 4 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.221727 29919 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.221864 29919 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 2 FOLLOWER]: Advancing to term 4
18:51:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:19.222219 29636 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:19.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.223166 29919 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 4.
18:51:19.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.223367 29634 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: fafc2bbe61c8493aa2bc1bfa367b392b
18:51:19.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.223469 29991 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 4 FOLLOWER]: Leader election won for term 4
18:51:19.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.223627 29991 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 4 LEADER]: Becoming Leader. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Running, Role: LEADER
18:51:19.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.223729 29991 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.224475 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 reported cstate change: term changed from 3 to 4, leader changed from fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130) to f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131). New cstate: current_term: 4 leader_uuid: "f8e2085855944ef6829cc9ccffc14be4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: UNKNOWN } } }
18:51:19.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:19.712318 29636 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 -> Peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Couldn't send request to peer fafc2bbe61c8493aa2bc1bfa367b392b. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.755635 29919 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 4 FOLLOWER]: Refusing update from remote peer f8e2085855944ef6829cc9ccffc14be4: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
18:51:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:19.756187 29991 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
18:51:20.356 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:20.357 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:20.357 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2476), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [4ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [14ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [17ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [20ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [21ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [34ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [36ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [37ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [37ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [74ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [77ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [79ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [80ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [112ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [114ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [117ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [119ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [119ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [492ms] refreshing cache from master, [492ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [495ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [497ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [500ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [500ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [732ms] refreshing cache from master, [732ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [734ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [738ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [740ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [740ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2472ms] refreshing cache from master, [2473ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [2474ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1961228486(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@61191911 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184) -> (continuation of Deferred@519349224 after retry RPC@1467392479) -> (continuation of Deferred@1674320139 after retry RPC@22422772) -> (continuation of Deferred@742111769 after retry RPC@512221712), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d559c7 -> wakeup thread Time-limited test -> (continuation of Deferred@2117600556 after retry RPC@1351446605) -> (continuation of Deferred@1173788461 after retry RPC@1716684625) -> (continuation of Deferred@202107207 after retry RPC@2115084184) -> (continuation of Deferred@519349224 after retry RPC@1467392479) -> (continuation of Deferred@1674320139 after retry RPC@22422772) -> (continuation of Deferred@742111769 after retry RPC@512221712))) after lookup
18:51:20.358 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:20.360 [DEBUG - kudu-nio-1] (Connection.java:757) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] Successfully connected to peer
18:51:20.380 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7b0f8553, L:/127.0.0.1:57892 - R:/127.28.7.131:33291] using TOKEN/null
18:51:20.391 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.130:46345
18:51:20.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:51:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.392577 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:46345
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45681
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.481168 30032 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.481405 30032 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.481428 30032 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.482635 30032 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.482703 30032 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.130
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.484000 30032 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:46345
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45681
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30032
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.484205 30032 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.484481 30032 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.490140 30037 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.490113 30040 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:20.490120 30038 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:20.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.491096 30032 server_base.cc:1034] running on GCE node
18:51:20.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.491360 30032 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:20.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.491591 30032 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.492756 30032 hybrid_clock.cc:648] HybridClock initialized: now 1786474280492727 us; error 42 us; skew 500 ppm
18:51:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.494273 30032 webserver.cc:460] Webserver started at http://127.28.7.130:45681/ using document root <none> and password file <none>
18:51:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.494478 30032 fs_manager.cc:359] Metadata directory not provided
18:51:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.494536 30032 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:20.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.495930 30032 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:51:20.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.496578 30047 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:20.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.496779 30032 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.496946 30032 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.497279 30032 fs_report.cc:383] FS layout report
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/data
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.511518 30032 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.511837 30032 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:20.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.511945 30032 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:20.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.512158 30032 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.512856 30054 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.514201 30032 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.514277 30032 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.514317 30032 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.515023 30032 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.515086 30032 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.515126 30054 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap starting.
18:51:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.522938 30032 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.130:46345
18:51:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.523039 30178 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.130:46345 every 8 connection(s)
18:51:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.523391 30054 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Log is configured to *not* fsync() on all Append() calls
18:51:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.523471 30032 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.526785 30054 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.527163 30054 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap complete.
18:51:20.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.528354 30054 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent bootstrapping tablet: real 0.013s	user 0.007s	sys 0.004s
18:51:20.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.528561 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 30032
18:51:20.529 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.531308 30181 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.531411 30181 heartbeater.cc:464] Registering TS with master...
18:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.531155 30054 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.531533 30054 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc2bbe61c8493aa2bc1bfa367b392b, State: Initialized, Role: FOLLOWER
18:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.531602 30181 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.531718 30054 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.532121 30054 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent starting tablet: real 0.004s	user 0.004s	sys 0.001s
18:51:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.532450 28753 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.532998 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.130:37945
18:51:20.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.533947 30181 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.534874 30180 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.534925 30180 heartbeater.cc:464] Registering TS with master...
18:51:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.534976 30180 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.535468 28929 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:20.535 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.536022 30182 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.536079 30182 heartbeater.cc:464] Registering TS with master...
18:51:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.536131 30182 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.536541 28839 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:20.545 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.131:33291
18:51:20.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:20.545636 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 29613
18:51:20.555 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:20.555 [DEBUG - kudu-nio-1] (Connection.java:239) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] handling channelInactive
18:51:20.555 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state READY due to: connection closed
18:51:20.573 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:20.576 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.576 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.576 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.596 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:20.596 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:20.597 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [1ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [21ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [40ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@589946505(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@32ab1718 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6fc09cb -> wakeup thread Time-limited test)) after lookup
18:51:20.597 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:20.600 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.600 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.600 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.636 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:20.636 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:20.637 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [1ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [21ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [40ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [42ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [45ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [45ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [80ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@589946505(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@32ab1718 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6fc09cb -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195))) after lookup
18:51:20.638 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:20.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.639 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:20.697 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:20.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=142), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [1ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [21ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [40ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [42ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [45ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [45ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [80ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [82ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [84ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [84ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [141ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@589946505(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@32ab1718 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6fc09cb -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081))) after lookup
18:51:20.699 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:20.702 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.702 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.702 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.876 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:20.876 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:20.877 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=321), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [1ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [21ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [40ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [42ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [45ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [45ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [80ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [82ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [84ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [84ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [141ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [143ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [147ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [148ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [318ms] refreshing cache from master, [318ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [320ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@589946505(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@32ab1718 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081) -> (continuation of Deferred@1971051510 after retry RPC@1744462223), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6fc09cb -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081) -> (continuation of Deferred@1971051510 after retry RPC@1744462223))) after lookup
18:51:20.878 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:20.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:20.879 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:21.176 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:21.177 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:21.178 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=622), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [1ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [21ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [40ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [42ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [45ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [45ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [80ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [82ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [84ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [84ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [141ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [143ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [147ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [148ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [318ms] refreshing cache from master, [318ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [320ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [322ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [324ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [324ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [618ms] refreshing cache from master, [618ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [620ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@589946505(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@32ab1718 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081) -> (continuation of Deferred@1971051510 after retry RPC@1744462223) -> (continuation of Deferred@1012951923 after retry RPC@1380433502), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6fc09cb -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081) -> (continuation of Deferred@1971051510 after retry RPC@1744462223) -> (continuation of Deferred@1012951923 after retry RPC@1380433502))) after lookup
18:51:21.178 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:21.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:21.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:21.181 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:21.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.657435 30197 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:51:21.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.657575 30197 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.657923 30197 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:21.658419 30048 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:21.658923 30048 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.661481 29919 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" is_pre_election: true
18:51:21.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.661908 30050 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4
18:51:21.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.662071 30197 raft_consensus.cc:2726] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
18:51:21.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.891635 30202 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:21.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.891774 30202 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:21.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.891973 30202 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:21.892848 29840 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.895591 30124 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" is_pre_election: true
18:51:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.895710 30124 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 3.
18:51:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.895920 29843 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:21.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.896019 30202 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 4 FOLLOWER]: Leader pre-election won for term 5
18:51:21.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.896088 30202 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 4 FOLLOWER]: Starting leader election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:21.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.896104 30202 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 4 FOLLOWER]: Advancing to term 5
18:51:21.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.896725 30202 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:21.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.896929 30202 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 5 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:21.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.897087 30124 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.897179 30124 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 3 FOLLOWER]: Advancing to term 5
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:21.897491 29840 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.898554 30124 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 5.
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.898759 29843 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.898883 30202 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 5 FOLLOWER]: Leader election won for term 5
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.899061 30202 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 5 LEADER]: Becoming Leader. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Running, Role: LEADER
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.899151 30202 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:21.899914 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 reported cstate change: term changed from 4 to 5, leader changed from f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131) to 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129). New cstate: current_term: 5 leader_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: HEALTHY } } }
18:51:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.351569 30124 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 5 FOLLOWER]: Refusing update from remote peer 2dc38ce1e08949da90122cd3ab2b4a31: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
18:51:22.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.351989 30202 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
18:51:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.355569 29840 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 -> Peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Couldn't send request to peer f8e2085855944ef6829cc9ccffc14be4. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:22.436 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:22.436 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:22.437 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1881), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [1ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [21ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [38ms] refreshing cache from master, [38ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [40ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [42ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [45ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [45ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [78ms] refreshing cache from master, [78ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [80ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [82ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [84ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [84ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [138ms] refreshing cache from master, [138ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [141ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [143ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [147ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [148ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [318ms] refreshing cache from master, [318ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [320ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [322ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [324ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [324ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [618ms] refreshing cache from master, [618ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [620ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [623ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [626ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [626ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [1878ms] refreshing cache from master, [1878ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [1880ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@589946505(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@32ab1718 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081) -> (continuation of Deferred@1971051510 after retry RPC@1744462223) -> (continuation of Deferred@1012951923 after retry RPC@1380433502) -> (continuation of Deferred@1318931770 after retry RPC@566738937), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6fc09cb -> wakeup thread Time-limited test -> (continuation of Deferred@590960926 after retry RPC@1164820195) -> (continuation of Deferred@678624794 after retry RPC@1936474081) -> (continuation of Deferred@1971051510 after retry RPC@1744462223) -> (continuation of Deferred@1012951923 after retry RPC@1380433502) -> (continuation of Deferred@1318931770 after retry RPC@566738937))) after lookup
18:51:22.438 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:22.439 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] Successfully connected to peer
18:51:22.454 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0d05fb75, L:/127.0.0.1:34836 - R:/127.28.7.129:43067] using TOKEN/null
18:51:22.466 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.131:33291
18:51:22.466 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.466846 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.555581 30234 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:22.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.555817 30234 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:22.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.555840 30234 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:22.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.556983 30234 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:22.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.557039 30234 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.131
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.558370 30234 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30234
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.558585 30234 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.558858 30234 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.564095 30239 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.564107 30240 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:22.564105 30242 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.564455 30234 server_base.cc:1034] running on GCE node
18:51:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.564683 30234 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.564998 30234 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:22.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.566166 30234 hybrid_clock.cc:648] HybridClock initialized: now 1786474282566137 us; error 48 us; skew 500 ppm
18:51:22.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.567533 30234 webserver.cc:460] Webserver started at http://127.28.7.131:33917/ using document root <none> and password file <none>
18:51:22.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.567735 30234 fs_manager.cc:359] Metadata directory not provided
18:51:22.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.567778 30234 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.569201 30234 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.569911 30249 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.570120 30234 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.570195 30234 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.570533 30234 fs_report.cc:383] FS layout report
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/data
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.579257 30234 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.579568 30234 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.579670 30234 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.579911 30234 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.580387 30258 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.581768 30234 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:22.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.581836 30234 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:22.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.581873 30234 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:22.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.582712 30234 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:22.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.582772 30234 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.582857 30258 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap starting.
18:51:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.590584 30234 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.131:33291
18:51:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.590938 30383 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.131:33291 every 8 connection(s)
18:51:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.591069 30234 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.591486 30258 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Log is configured to *not* fsync() on all Append() calls
18:51:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.592304 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 30234
18:51:22.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.595523 30258 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.595865 30258 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap complete.
18:51:22.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.597146 30258 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent bootstrapping tablet: real 0.014s	user 0.002s	sys 0.011s
18:51:22.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.598533 30258 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.598884 30258 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Initialized, Role: FOLLOWER
18:51:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.599069 30258 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.599480 30258 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.002s
18:51:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.599586 30385 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.599681 30385 heartbeater.cc:464] Registering TS with master...
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.599886 30385 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.600035 30386 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.600215 30384 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.600256 30384 heartbeater.cc:464] Registering TS with master...
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.600313 30384 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.600201 30386 heartbeater.cc:464] Registering TS with master...
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.600508 28753 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.600651 28923 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:22.600 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:97)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
18:51:22.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.601060 30386 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.601046 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.131:40869
18:51:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.601378 28839 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.601965 30385 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:22.602 [ERROR - Time-limited test] (ResourceLeakDetector.java:327) LEAK: HashedWheelTimer.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records: 
Created at:
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:312)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:251)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:224)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:203)
	io.netty.util.HashedWheelTimer.<init>(HashedWheelTimer.java:185)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:296)
	org.apache.kudu.client.AsyncKuduScanner.<init>(AsyncKuduScanner.java:431)
	org.apache.kudu.client.AsyncKuduScanner$AsyncKuduScannerBuilder.build(AsyncKuduScanner.java:1417)
	org.apache.kudu.client.TestMultipleLeaderFailover$1.get(TestMultipleLeaderFailover.java:48)
	org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
	org.apache.kudu.client.TestMultipleLeaderFailover.waitUntilRowCount(TestMultipleLeaderFailover.java:44)
	org.apache.kudu.client.TestMultipleLeaderFailover.testMultipleFailover(TestMultipleLeaderFailover.java:79)
	sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	java.lang.reflect.Method.invoke(Method.java:498)
	org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	java.util.concurrent.FutureTask.run(FutureTask.java:266)
	java.lang.Thread.run(Thread.java:750)
18:51:22.606 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:22.608 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.129:43067
18:51:22.608 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:51:22.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:22.608596 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 29821
18:51:22.617 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:22.617 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] handling channelInactive
18:51:22.617 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state READY due to: connection closed
18:51:22.617 [DEBUG - Time-limited test] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:22.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.620 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.636 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:22.637 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:22.637 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [3ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@590375752(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@1c887133 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f8a78b4 -> wakeup thread Time-limited test)) after lookup
18:51:22.638 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:22.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.639 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.656 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:22.656 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:22.657 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [3ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [21ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [22ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@590375752(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@1c887133 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f8a78b4 -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805))) after lookup
18:51:22.657 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:22.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.660 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.696 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:22.696 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:22.697 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [3ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [21ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [22ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [42ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [43ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [78ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@590375752(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@1c887133 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f8a78b4 -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003))) after lookup
18:51:22.698 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:22.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.699 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.736 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:22.736 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:22.737 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=120), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [3ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [21ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [22ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [42ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [43ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [78ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [80ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [82ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [83ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [118ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@590375752(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@1c887133 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003) -> (continuation of Deferred@494562902 after retry RPC@1808321502), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f8a78b4 -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003) -> (continuation of Deferred@494562902 after retry RPC@1808321502))) after lookup
18:51:22.738 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:22.740 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.741 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.796 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:22.796 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:22.797 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=179), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [3ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [21ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [22ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [42ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [43ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [78ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [80ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [82ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [83ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [118ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [120ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [123ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [124ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [178ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@590375752(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@1c887133 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003) -> (continuation of Deferred@494562902 after retry RPC@1808321502) -> (continuation of Deferred@340550729 after retry RPC@972544805), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f8a78b4 -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003) -> (continuation of Deferred@494562902 after retry RPC@1808321502) -> (continuation of Deferred@340550729 after retry RPC@972544805))) after lookup
18:51:22.798 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:22.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:22.799 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.964736 30402 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 2dc38ce1e08949da90122cd3ab2b4a31)
18:51:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.964855 30402 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.965073 30402 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:23.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:23.965960 30050 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:23.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.968910 30327 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "f8e2085855944ef6829cc9ccffc14be4" is_pre_election: true
18:51:23.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.969112 30327 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc2bbe61c8493aa2bc1bfa367b392b in term 4.
18:51:23.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.969341 30048 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: f8e2085855944ef6829cc9ccffc14be4, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31
18:51:23.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.969470 30402 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 5 FOLLOWER]: Leader pre-election won for term 6
18:51:23.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.969539 30402 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 5 FOLLOWER]: Starting leader election (detected failure of leader 2dc38ce1e08949da90122cd3ab2b4a31)
18:51:23.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.969553 30402 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 5 FOLLOWER]: Advancing to term 6
18:51:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.970180 30402 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.970322 30402 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 6 election: Requested vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.970485 30327 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.970595 30327 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 4 FOLLOWER]: Advancing to term 6
18:51:23.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:23.970794 30050 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.971899 30327 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc2bbe61c8493aa2bc1bfa367b392b in term 6.
18:51:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.972163 30048 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: f8e2085855944ef6829cc9ccffc14be4, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31
18:51:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.972297 30402 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 6 FOLLOWER]: Leader election won for term 6
18:51:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.972443 30402 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 6 LEADER]: Becoming Leader. State: Replica: fafc2bbe61c8493aa2bc1bfa367b392b, State: Running, Role: LEADER
18:51:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.972553 30402 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:23.973480 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b reported cstate change: term changed from 5 to 6, leader changed from 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129) to fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130). New cstate: current_term: 6 leader_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: UNKNOWN } } }
18:51:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.356293 30050 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b -> Peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Couldn't send request to peer 2dc38ce1e08949da90122cd3ab2b4a31. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.510291 30327 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 6 FOLLOWER]: Refusing update from remote peer fafc2bbe61c8493aa2bc1bfa367b392b: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
18:51:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.510749 30402 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
18:51:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.512230 30423 mvcc.cc:205] Tried to move back new op lower bound from 7317398660938756096 to 7317398658659495936. Current Snapshot: MvccSnapshot[applied={T|T < 7317398660938756096}]
18:51:24.596 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:24.596 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:24.597 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1979), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [3ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [3ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [18ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [21ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [22ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [22ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [38ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [40ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [42ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [43ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [78ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [80ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [82ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [83ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [118ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [120ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [123ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [124ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [176ms] refreshing cache from master, [176ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [178ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [180ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [182ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [182ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [1976ms] refreshing cache from master, [1976ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [1978ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@590375752(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@1c887133 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003) -> (continuation of Deferred@494562902 after retry RPC@1808321502) -> (continuation of Deferred@340550729 after retry RPC@972544805) -> (continuation of Deferred@661114525 after retry RPC@1635261585), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f8a78b4 -> wakeup thread Time-limited test -> (continuation of Deferred@853872456 after retry RPC@1449484805) -> (continuation of Deferred@743489942 after retry RPC@1364786003) -> (continuation of Deferred@494562902 after retry RPC@1808321502) -> (continuation of Deferred@340550729 after retry RPC@972544805) -> (continuation of Deferred@661114525 after retry RPC@1635261585))) after lookup
18:51:24.598 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:24.600 [DEBUG - kudu-nio-1] (Connection.java:757) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] Successfully connected to peer
18:51:24.620 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xafc09d5c, L:/127.0.0.1:52186 - R:/127.28.7.130:46345] using TOKEN/null
18:51:24.636 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.129:43067
18:51:24.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.638091 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:43067
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.723134 30433 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.723418 30433 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.723457 30433 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.724756 30433 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.724918 30433 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.129
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.726538 30433 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:43067
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30433
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.726830 30433 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.727221 30433 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:24.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.733315 30439 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:24.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.733336 30440 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:24.733314 30442 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.734050 30433 server_base.cc:1034] running on GCE node
18:51:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.734272 30433 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.734536 30433 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.735730 30433 hybrid_clock.cc:648] HybridClock initialized: now 1786474284735706 us; error 43 us; skew 500 ppm
18:51:24.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.741005 30433 webserver.cc:460] Webserver started at http://127.28.7.129:37913/ using document root <none> and password file <none>
18:51:24.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.741324 30433 fs_manager.cc:359] Metadata directory not provided
18:51:24.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.741420 30433 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.742870 30433 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.002s
18:51:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.743811 30449 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.743984 30433 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.744087 30433 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.744516 30433 fs_report.cc:383] FS layout report
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/data
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:24.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.758412 30433 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.758891 30433 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.759106 30433 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.759449 30433 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.760048 30457 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.761631 30433 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.761809 30433 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.761873 30433 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.762697 30433 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.762785 30433 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.762905 30457 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap starting.
18:51:24.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.770817 30433 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.129:43067
18:51:24.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.770905 30587 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.129:43067 every 8 connection(s)
18:51:24.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.771366 30433 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.773871 30457 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Log is configured to *not* fsync() on all Append() calls
18:51:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.774088 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 30433
18:51:24.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:24.780 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.780802 30593 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.780962 30593 heartbeater.cc:464] Registering TS with master...
18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.781068 30591 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.781131 30591 heartbeater.cc:464] Registering TS with master...
18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.781172 30593 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.781214 30591 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:24.781 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.130:46345
18:51:24.781 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.781738 30457 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.781809 28753 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.781925 28842 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:24.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.782162 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 30032
18:51:24.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.782374 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.129:38113
18:51:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.786885 30589 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.786998 30589 heartbeater.cc:464] Registering TS with master...
18:51:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.787101 30589 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.787343 30457 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap complete.
18:51:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.787568 28923 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.788398 30457 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent bootstrapping tablet: real 0.026s	user 0.009s	sys 0.006s
18:51:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.789561 30457 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.789846 30457 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Initialized, Role: FOLLOWER
18:51:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.790005 30457 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.790349 30457 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent starting tablet: real 0.002s	user 0.000s	sys 0.003s
18:51:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:24.790416 30591 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:24.793 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:24.793 [DEBUG - kudu-nio-1] (Connection.java:239) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] handling channelInactive
18:51:24.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state READY due to: connection closed
18:51:24.794 [DEBUG - Time-limited test] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:24.796 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.796 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.796 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.815 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:24.815 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:24.816 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [3ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [21ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@580399955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@2d7c0b21 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6adf15b6 -> wakeup thread Time-limited test)) after lookup
18:51:24.816 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:24.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.818 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.835 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:24.836 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:24.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [3ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [21ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [23ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [25ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [26ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [42ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@580399955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@2d7c0b21 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6adf15b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190))) after lookup
18:51:24.837 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:24.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.838 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.855 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:24.855 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:24.855 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [3ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [21ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [23ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [25ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [26ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [42ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [45ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [45ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [59ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [61ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@580399955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@2d7c0b21 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6adf15b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555))) after lookup
18:51:24.856 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:24.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.858 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.895 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:24.895 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:24.896 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=102), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [3ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [21ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [23ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [25ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [26ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [42ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [45ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [45ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [59ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [61ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [63ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [65ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [66ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [102ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@580399955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@2d7c0b21 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555) -> (continuation of Deferred@235589010 after retry RPC@529136738), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6adf15b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555) -> (continuation of Deferred@235589010 after retry RPC@529136738))) after lookup
18:51:24.897 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:24.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:24.898 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:25.255 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:25.256 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:25.256 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=462), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [3ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [21ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [23ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [25ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [26ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [42ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [45ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [45ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [59ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [61ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [63ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [65ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [66ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [102ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [104ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [105ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [105ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [460ms] refreshing cache from master, [460ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [462ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@580399955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@2d7c0b21 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555) -> (continuation of Deferred@235589010 after retry RPC@529136738) -> (continuation of Deferred@404820762 after retry RPC@1541140220), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6adf15b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555) -> (continuation of Deferred@235589010 after retry RPC@529136738) -> (continuation of Deferred@404820762 after retry RPC@1541140220))) after lookup
18:51:25.257 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:25.259 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:25.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:25.259 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:26.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.135421 30608 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader fafc2bbe61c8493aa2bc1bfa367b392b)
18:51:26.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.135519 30608 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:26.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.135830 30608 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:26.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.136303 30254 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.136868 30254 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:26.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.140311 30530 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" is_pre_election: true
18:51:26.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.140501 30530 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 5.
18:51:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.140789 30253 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: fafc2bbe61c8493aa2bc1bfa367b392b
18:51:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.140995 30608 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 6 FOLLOWER]: Leader pre-election won for term 7
18:51:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.141050 30608 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 6 FOLLOWER]: Starting leader election (detected failure of leader fafc2bbe61c8493aa2bc1bfa367b392b)
18:51:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.141077 30608 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 6 FOLLOWER]: Advancing to term 7
18:51:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.141738 30608 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.141907 30608 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 7 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.142164 30530 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.142273 30530 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 5 FOLLOWER]: Advancing to term 7
18:51:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.142519 30254 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:26.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.143607 30530 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 7.
18:51:26.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.143811 30253 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: fafc2bbe61c8493aa2bc1bfa367b392b
18:51:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.143946 30608 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Leader election won for term 7
18:51:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.144097 30608 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 LEADER]: Becoming Leader. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Running, Role: LEADER
18:51:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.144186 30608 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:26.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.144927 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 reported cstate change: term changed from 6 to 7, leader changed from fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130) to f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131). New cstate: current_term: 7 leader_uuid: "f8e2085855944ef6829cc9ccffc14be4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: UNKNOWN } } }
18:51:26.495 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:26.496 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:26.496 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1702), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [3ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [3ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [21ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [23ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [25ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [26ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [42ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [45ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [45ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [59ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [61ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [63ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [65ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [66ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [99ms] refreshing cache from master, [100ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [102ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [104ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [105ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [105ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [460ms] refreshing cache from master, [460ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [462ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [464ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [466ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [467ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [1700ms] refreshing cache from master, [1700ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [1702ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@580399955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@2d7c0b21 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555) -> (continuation of Deferred@235589010 after retry RPC@529136738) -> (continuation of Deferred@404820762 after retry RPC@1541140220) -> (continuation of Deferred@1331730830 after retry RPC@1263018640), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6adf15b6 -> wakeup thread Time-limited test -> (continuation of Deferred@1248895651 after retry RPC@1485864190) -> (continuation of Deferred@865382296 after retry RPC@616999555) -> (continuation of Deferred@235589010 after retry RPC@529136738) -> (continuation of Deferred@404820762 after retry RPC@1541140220) -> (continuation of Deferred@1331730830 after retry RPC@1263018640))) after lookup
18:51:26.497 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:26.498 [DEBUG - kudu-nio-0] (Connection.java:757) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] Successfully connected to peer
18:51:26.518 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4d6ac89e, L:/127.0.0.1:57956 - R:/127.28.7.131:33291] using TOKEN/null
18:51:26.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.520082 30530 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 7 FOLLOWER]: Refusing update from remote peer f8e2085855944ef6829cc9ccffc14be4: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
18:51:26.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.520515 30613 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
18:51:26.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.520874 30254 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 -> Peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Couldn't send request to peer fafc2bbe61c8493aa2bc1bfa367b392b. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.524068 30633 mvcc.cc:205] Tried to move back new op lower bound from 7317398677583421440 to 7317398676047126528. Current Snapshot: MvccSnapshot[applied={T|T < 7317398677583421440}]
18:51:26.532 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.130:46345
18:51:26.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.533272 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:46345
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45681
18:51:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:26.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.647547 30642 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:26.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.647836 30642 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:26.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.647874 30642 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.649309 30642 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.649400 30642 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.130
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.650862 30642 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:46345
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45681
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30642
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.651167 30642 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.651559 30642 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:26.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.658210 30647 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:26.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.658229 30651 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:26.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:26.658229 30648 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:26.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.658270 30642 server_base.cc:1034] running on GCE node
18:51:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.658949 30642 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.659162 30642 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.660344 30642 hybrid_clock.cc:648] HybridClock initialized: now 1786474286660314 us; error 48 us; skew 500 ppm
18:51:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.661976 30642 webserver.cc:460] Webserver started at http://127.28.7.130:45681/ using document root <none> and password file <none>
18:51:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.662284 30642 fs_manager.cc:359] Metadata directory not provided
18:51:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.662366 30642 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:26.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.663864 30642 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.002s
18:51:26.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.664664 30659 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:26.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.664845 30642 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.664937 30642 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.665272 30642 fs_report.cc:383] FS layout report
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/data
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.692446 30642 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.692765 30642 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.692971 30642 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.693202 30642 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.693778 30670 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.695600 30642 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.695676 30642 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.695715 30642 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.696455 30642 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.696509 30642 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.696585 30670 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap starting.
18:51:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.704371 30642 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.130:46345
18:51:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.704466 30797 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.130:46345 every 8 connection(s)
18:51:26.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.704803 30642 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:26.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.707209 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 30642
18:51:26.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:26.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.707199 30670 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Log is configured to *not* fsync() on all Append() calls
18:51:26.713 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:26.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.713174 30800 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:26.713 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.131:33291
18:51:26.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.713289 30800 heartbeater.cc:464] Registering TS with master...
18:51:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.713496 30800 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:26.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.714272 28842 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.714409 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 30234
18:51:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.720257 30798 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.720366 30798 heartbeater.cc:464] Registering TS with master...
18:51:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.720460 30798 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.721017 28923 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.722823 30799 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.722873 30799 heartbeater.cc:464] Registering TS with master...
18:51:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.722963 30799 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.723243 28751 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.723824 28751 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.130:35783
18:51:26.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.726759 30670 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:26.727 [DEBUG - kudu-nio-0] (Connection.java:239) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] handling channelInactive
18:51:26.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:26.727 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state READY due to: connection closed
18:51:26.727 [DEBUG - Time-limited test] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.727149 30670 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap complete.
18:51:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.728207 30670 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent bootstrapping tablet: real 0.032s	user 0.005s	sys 0.009s
18:51:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.729504 30670 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.729904 30670 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc2bbe61c8493aa2bc1bfa367b392b, State: Initialized, Role: FOLLOWER
18:51:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.730017 30670 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.730320 30670 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent starting tablet: real 0.002s	user 0.004s	sys 0.000s
18:51:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:26.730607 30799 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:26.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.732 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.755 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:26.756 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:26.756 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [5ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [5ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [28ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@736016230(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@273437ab -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38a71765 -> wakeup thread Time-limited test)) after lookup
18:51:26.757 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:26.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.758 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.776 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:26.776 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:26.776 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [5ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [5ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [28ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [29ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [31ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [31ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [48ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@736016230(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@273437ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38a71765 -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386))) after lookup
18:51:26.778 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:26.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.780 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.795 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:26.796 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:26.796 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [5ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [5ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [28ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [29ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [31ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [31ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [48ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [50ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [53ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [53ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [68ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@736016230(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@273437ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38a71765 -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057))) after lookup
18:51:26.797 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:26.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.798 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.915 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:26.916 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:26.916 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=188), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [5ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [5ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [28ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [29ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [31ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [31ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [48ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [50ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [53ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [53ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [68ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [70ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [71ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [71ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [188ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@736016230(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@273437ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38a71765 -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568))) after lookup
18:51:26.917 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:26.919 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.919 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:26.919 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:27.175 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:27.176 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:27.176 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=448), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [5ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [5ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [28ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [29ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [31ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [31ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [48ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [50ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [53ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [53ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [68ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [70ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [71ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [71ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [188ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [190ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [192ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [192ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [445ms] refreshing cache from master, [446ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [448ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@736016230(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@273437ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568) -> (continuation of Deferred@1861453184 after retry RPC@2015701981), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38a71765 -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568) -> (continuation of Deferred@1861453184 after retry RPC@2015701981))) after lookup
18:51:27.177 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:27.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:27.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:27.178 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:27.716 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:27.716 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:27.717 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=989), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [5ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [5ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [28ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [29ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [31ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [31ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [48ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [50ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [53ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [53ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [68ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [70ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [71ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [71ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [188ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [190ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [192ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [192ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [445ms] refreshing cache from master, [446ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [448ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [449ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [451ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [451ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [986ms] refreshing cache from master, [986ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [988ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@736016230(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@273437ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568) -> (continuation of Deferred@1861453184 after retry RPC@2015701981) -> (continuation of Deferred@200123279 after retry RPC@301990501), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38a71765 -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568) -> (continuation of Deferred@1861453184 after retry RPC@2015701981) -> (continuation of Deferred@200123279 after retry RPC@301990501))) after lookup
18:51:27.717 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:27.719 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:27.719 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:27.720 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.032148 30815 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.032250 30815 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.032626 30815 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:28.033198 30450 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:28.033655 30450 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.037158 30745 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" is_pre_election: true
18:51:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.037312 30745 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 6.
18:51:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.037585 30453 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.037720 30815 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 7 FOLLOWER]: Leader pre-election won for term 8
18:51:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.037781 30815 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 7 FOLLOWER]: Starting leader election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.037801 30815 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 7 FOLLOWER]: Advancing to term 8
18:51:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.038444 30815 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.038607 30815 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 8 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.038775 30745 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.038874 30745 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 6 FOLLOWER]: Advancing to term 8
18:51:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:28.039302 30450 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.040087 30745 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 8.
18:51:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.040287 30453 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.040418 30815 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 8 FOLLOWER]: Leader election won for term 8
18:51:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.040637 30815 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 8 LEADER]: Becoming Leader. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Running, Role: LEADER
18:51:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.040736 30815 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.041592 28751 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 reported cstate change: term changed from 7 to 8, leader changed from f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131) to 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129). New cstate: current_term: 8 leader_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: HEALTHY } } }
18:51:28.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:28.442658 30450 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 -> Peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Couldn't send request to peer f8e2085855944ef6829cc9ccffc14be4. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.618801 30745 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 8 FOLLOWER]: Refusing update from remote peer 2dc38ce1e08949da90122cd3ab2b4a31: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
18:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.619208 30815 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
18:51:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:28.620747 30834 mvcc.cc:205] Tried to move back new op lower bound from 7317398677583421440 to 7317398676047126528. Current Snapshot: MvccSnapshot[applied={T|T < 7317398677583421440}]
18:51:30.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:30.861670 30450 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 -> Peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Couldn't send request to peer f8e2085855944ef6829cc9ccffc14be4. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:51:31.095 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:31.095 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:31.095 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=4368), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [5ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [5ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [28ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [29ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [31ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [31ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [48ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [50ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [53ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [53ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [66ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [68ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [70ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [71ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [71ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [186ms] refreshing cache from master, [186ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [188ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [190ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [192ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [192ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [445ms] refreshing cache from master, [446ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [448ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [449ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [451ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [451ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [986ms] refreshing cache from master, [986ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [988ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [990ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [993ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [993ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [4365ms] refreshing cache from master, [4365ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [4367ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@736016230(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@273437ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568) -> (continuation of Deferred@1861453184 after retry RPC@2015701981) -> (continuation of Deferred@200123279 after retry RPC@301990501) -> (continuation of Deferred@1889596049 after retry RPC@1654773194), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@38a71765 -> wakeup thread Time-limited test -> (continuation of Deferred@1374575376 after retry RPC@146143386) -> (continuation of Deferred@1255022906 after retry RPC@1838703057) -> (continuation of Deferred@326872823 after retry RPC@1306227568) -> (continuation of Deferred@1861453184 after retry RPC@2015701981) -> (continuation of Deferred@200123279 after retry RPC@301990501) -> (continuation of Deferred@1889596049 after retry RPC@1654773194))) after lookup
18:51:31.096 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:31.097 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] Successfully connected to peer
18:51:31.113 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7b971d65, L:/127.0.0.1:34594 - R:/127.28.7.129:43067] using TOKEN/null
18:51:31.123 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.131:33291
18:51:31.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.123814 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.216950 30860 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.217186 30860 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.217223 30860 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:31.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.218523 30860 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:31.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.218622 30860 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.131
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.220201 30860 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.30860
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.220480 30860 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.220901 30860 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.226475 30867 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.226475 30866 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:31.226579 30869 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.226608 30860 server_base.cc:1034] running on GCE node
18:51:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.227079 30860 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.227283 30860 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:31.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.228430 30860 hybrid_clock.cc:648] HybridClock initialized: now 1786474291228410 us; error 41 us; skew 500 ppm
18:51:31.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.235023 30860 webserver.cc:460] Webserver started at http://127.28.7.131:33917/ using document root <none> and password file <none>
18:51:31.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.235355 30860 fs_manager.cc:359] Metadata directory not provided
18:51:31.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.235440 30860 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.237064 30860 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.002s
18:51:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.237761 30878 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.238015 30860 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.238114 30860 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.238515 30860 fs_report.cc:383] FS layout report
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/data
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.250522 30860 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.250931 30860 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.251109 30860 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.251485 30860 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.252054 30886 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.253742 30860 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.253810 30860 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.253847 30860 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:31.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.254532 30860 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.254590 30860 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.254676 30886 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap starting.
18:51:31.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.261623 30860 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.131:33291
18:51:31.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.261684 31008 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.131:33291 every 8 connection(s)
18:51:31.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.262068 30860 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.264932 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 30860
18:51:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.265044 30886 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Log is configured to *not* fsync() on all Append() calls
18:51:31.265 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.269934 31011 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.270051 31011 heartbeater.cc:464] Registering TS with master...
18:51:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.270262 31011 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.270444 31010 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.270479 31010 heartbeater.cc:464] Registering TS with master...
18:51:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.270532 31010 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.270820 28753 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:31.271 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.271294 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.131:44971
18:51:31.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.271704 28923 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:31.272 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.129:43067
18:51:31.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:51:31.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.273063 31012 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:31.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.273118 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 30433
18:51:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.275429 31012 heartbeater.cc:464] Registering TS with master...
18:51:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.275664 31012 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.276165 30886 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.276216 28839 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.276541 30886 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap complete.
18:51:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.277571 30886 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent bootstrapping tablet: real 0.023s	user 0.014s	sys 0.001s
18:51:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.278594 30886 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.278853 30886 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Initialized, Role: FOLLOWER
18:51:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.278985 30886 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.279263 31011 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:31.279263 30886 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent starting tablet: real 0.002s	user 0.001s	sys 0.002s
18:51:31.282 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:31.282 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] handling channelInactive
18:51:31.283 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state READY due to: connection closed
18:51:31.283 [DEBUG - Time-limited test] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:31.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.285 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.295 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:31.295 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:31.295 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [11ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1569239178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@7b45d0ab -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fc2b86d -> wakeup thread Time-limited test)) after lookup
18:51:31.296 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:31.297 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.297 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.297 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.314 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:31.315 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:31.315 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [11ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [14ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [31ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1569239178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@7b45d0ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fc2b86d -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926))) after lookup
18:51:31.316 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:31.317 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.317 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.317 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.335 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:31.335 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:31.335 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [11ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [14ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [31ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [32ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [34ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [51ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1569239178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@7b45d0ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fc2b86d -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852))) after lookup
18:51:31.336 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:31.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.338 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.355 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:31.356 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:31.356 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [11ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [14ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [31ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [32ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [34ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [51ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [53ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [55ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [55ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [72ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1569239178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@7b45d0ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852) -> (continuation of Deferred@604943607 after retry RPC@1669966004), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fc2b86d -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852) -> (continuation of Deferred@604943607 after retry RPC@1669966004))) after lookup
18:51:31.357 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:31.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.359 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.676 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:31.676 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:31.676 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=393), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [11ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [14ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [31ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [32ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [34ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [51ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [53ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [55ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [55ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [72ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [74ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [76ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [76ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [390ms] refreshing cache from master, [390ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [392ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1569239178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@7b45d0ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852) -> (continuation of Deferred@604943607 after retry RPC@1669966004) -> (continuation of Deferred@1336343104 after retry RPC@1631856392), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fc2b86d -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852) -> (continuation of Deferred@604943607 after retry RPC@1669966004) -> (continuation of Deferred@1336343104 after retry RPC@1631856392))) after lookup
18:51:31.677 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:31.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:31.679 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067
18:51:32.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.597941 31024 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:51:32.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.598080 31024 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:32.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.598457 31024 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:32.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:32.598989 30882 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:32.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:32.599581 30882 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.602914 30745 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" is_pre_election: true
18:51:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.603226 30883 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f8e2085855944ef6829cc9ccffc14be4; no voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b
18:51:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.603374 31024 raft_consensus.cc:2726] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
18:51:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.623510 31031 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 2dc38ce1e08949da90122cd3ab2b4a31)
18:51:32.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.623726 31031 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:32.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.624203 31031 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:32.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:32.624703 30662 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:32.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:32.625293 30662 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.628145 30957 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "f8e2085855944ef6829cc9ccffc14be4" is_pre_election: true
18:51:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.628317 30957 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fafc2bbe61c8493aa2bc1bfa367b392b in term 7.
18:51:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.628561 30660 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: f8e2085855944ef6829cc9ccffc14be4, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31
18:51:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.628710 31031 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 8 FOLLOWER]: Leader pre-election won for term 9
18:51:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.628769 31031 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 8 FOLLOWER]: Starting leader election (detected failure of leader 2dc38ce1e08949da90122cd3ab2b4a31)
18:51:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.628785 31031 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 8 FOLLOWER]: Advancing to term 9
18:51:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.629446 31031 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.629585 31031 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 9 election: Requested vote from peers f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.629738 30957 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.629855 30957 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 7 FOLLOWER]: Advancing to term 9
18:51:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:32.630048 30662 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111)
18:51:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.630991 30957 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fafc2bbe61c8493aa2bc1bfa367b392b in term 9.
18:51:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.631146 30660 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: f8e2085855944ef6829cc9ccffc14be4, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: 2dc38ce1e08949da90122cd3ab2b4a31
18:51:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.631237 31031 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 9 FOLLOWER]: Leader election won for term 9
18:51:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.631407 31031 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 9 LEADER]: Becoming Leader. State: Replica: fafc2bbe61c8493aa2bc1bfa367b392b, State: Running, Role: LEADER
18:51:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.631486 31031 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:32.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.632134 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b reported cstate change: term changed from 8 to 9, leader changed from 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129) to fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130). New cstate: current_term: 9 leader_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: UNKNOWN } } }
18:51:32.915 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:32.915 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:32.915 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1632), Traces: [0ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [2ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [9ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [11ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [14ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [31ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [32ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [34ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [34ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [51ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [53ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [55ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [55ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [72ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [74ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [76ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [76ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [390ms] refreshing cache from master, [390ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [392ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [394ms] sending RPC to server 2dc38ce1e08949da90122cd3ab2b4a31, [396ms] received response from server 2dc38ce1e08949da90122cd3ab2b4a31: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [396ms] delaying RPC due to: Network error: Failed to connect to peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067): Connection refused: /127.28.7.129:43067, [1629ms] refreshing cache from master, [1629ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [1631ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1569239178(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@7b45d0ab -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852) -> (continuation of Deferred@604943607 after retry RPC@1669966004) -> (continuation of Deferred@1336343104 after retry RPC@1631856392) -> (continuation of Deferred@1597878493 after retry RPC@1821378124), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4fc2b86d -> wakeup thread Time-limited test -> (continuation of Deferred@634105139 after retry RPC@2099368926) -> (continuation of Deferred@20040687 after retry RPC@1920812852) -> (continuation of Deferred@604943607 after retry RPC@1669966004) -> (continuation of Deferred@1336343104 after retry RPC@1631856392) -> (continuation of Deferred@1597878493 after retry RPC@1821378124))) after lookup
18:51:32.916 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:32.918 [DEBUG - kudu-nio-1] (Connection.java:757) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] Successfully connected to peer
18:51:32.932 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x148abd32, L:/127.0.0.1:48528 - R:/127.28.7.130:46345] using TOKEN/null
18:51:32.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.934425 30957 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 9 FOLLOWER]: Refusing update from remote peer fafc2bbe61c8493aa2bc1bfa367b392b: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 34. (index mismatch)
18:51:32.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:32.934598 30662 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b -> Peer 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067): Couldn't send request to peer 2dc38ce1e08949da90122cd3ab2b4a31. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.129:43067: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:32.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.934808 31037 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
18:51:32.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.944380 31054 mvcc.cc:205] Tried to move back new op lower bound from 7317398703856398336 to 7317398702619066368. Current Snapshot: MvccSnapshot[applied={T|T < 7317398703856398336 or (T in {7317398703856398336})}]
18:51:32.950 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.129:43067
18:51:32.950 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:51:32.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:32.950798 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:32.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:43067
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:33.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.038700 31061 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:33.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.038941 31061 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:33.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.038964 31061 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:33.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.040073 31061 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:33.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.040122 31061 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.129
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.041586 31061 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.129:43067
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.129
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31061
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.129
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/logs
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.041807 31061 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:33.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.042137 31061 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.047931 31068 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.047983 31070 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:33.048137 31067 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.048403 31061 server_base.cc:1034] running on GCE node
18:51:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.048651 31061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:33.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.048950 31061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.050107 31061 hybrid_clock.cc:648] HybridClock initialized: now 1786474293050094 us; error 31 us; skew 500 ppm
18:51:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.051339 31061 webserver.cc:460] Webserver started at http://127.28.7.129:37913/ using document root <none> and password file <none>
18:51:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.051676 31061 fs_manager.cc:359] Metadata directory not provided
18:51:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.051746 31061 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.053567 31061 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.002s
18:51:33.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.054378 31077 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:33.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.054577 31061 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.054663 31061 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.055027 31061 fs_report.cc:383] FS layout report
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/wal
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/data
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.065873 31061 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.066165 31061 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.066269 31061 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.066479 31061 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.067041 31084 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.068425 31061 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.068532 31061 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.068583 31061 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.069588 31061 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.069674 31061 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.069698 31084 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap starting.
18:51:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.077847 31061 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.129:43067
18:51:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.077973 31213 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.129:43067 every 8 connection(s)
18:51:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.078378 31061 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-0/data/info.pb
18:51:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.079878 31084 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Log is configured to *not* fsync() on all Append() calls
18:51:33.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.084746 31084 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:33.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.085145 31084 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Bootstrap complete.
18:51:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.085918 31218 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.086035 31218 heartbeater.cc:464] Registering TS with master...
18:51:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.086217 31218 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.086298 31084 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent bootstrapping tablet: real 0.017s	user 0.003s	sys 0.012s
18:51:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.086457 31219 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.086509 31219 heartbeater.cc:464] Registering TS with master...
18:51:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.086563 31219 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.087044 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 31061
18:51:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.087314 28839 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:33.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.087482 28753 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.087626 31084 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.087937 31084 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Initialized, Role: FOLLOWER
18:51:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.087961 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.129:42507
18:51:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.088061 31084 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.088366 31084 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31: Time spent starting tablet: real 0.002s	user 0.001s	sys 0.003s
18:51:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.090013 31218 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.092264 31216 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.092336 31216 heartbeater.cc:464] Registering TS with master...
18:51:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.092396 31216 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.092746 28923 ts_manager.cc:194] Re-registered known tserver with Master: 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:33.092 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:33.093 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.130:46345
18:51:33.093 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:51:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:33.093974 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 30642
18:51:33.102 [DEBUG - kudu-nio-1] (Connection.java:239) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] handling channelInactive
18:51:33.102 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:33.102 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state READY due to: connection closed
18:51:33.103 [DEBUG - Time-limited test] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:33.104 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.104 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.104 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.114 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:33.115 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:33.115 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [2ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [12ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@235864756(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@26d3d257 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@732b94f3 -> wakeup thread Time-limited test)) after lookup
18:51:33.115 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:33.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.117 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.135 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:33.136 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:33.136 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [2ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [12ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [15ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [15ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@235864756(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@26d3d257 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@732b94f3 -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859))) after lookup
18:51:33.137 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:33.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.138 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.175 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:33.175 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:33.176 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [2ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [12ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [15ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [15ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [35ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [36ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [36ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [73ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@235864756(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@26d3d257 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@732b94f3 -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061))) after lookup
18:51:33.176 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:33.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.178 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.235 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:33.235 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:33.236 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=133), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [2ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [12ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [15ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [15ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [35ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [36ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [36ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [73ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [74ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [76ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [76ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [133ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@235864756(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@26d3d257 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@732b94f3 -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873))) after lookup
18:51:33.237 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:33.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.238 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.555 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:33.555 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:33.556 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=453), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [2ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [12ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [15ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [15ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [35ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [36ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [36ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [73ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [74ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [76ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [76ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [133ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [134ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [136ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [136ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [451ms] refreshing cache from master, [451ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [453ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@235864756(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@26d3d257 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873) -> (continuation of Deferred@408599516 after retry RPC@1331091850), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@732b94f3 -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873) -> (continuation of Deferred@408599516 after retry RPC@1331091850))) after lookup
18:51:33.556 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:33.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.559 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.955 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:33.955 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)[L]]}]
18:51:33.956 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=853), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [2ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [12ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [15ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [15ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [35ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [36ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [36ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [73ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [74ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [76ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [76ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [133ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [134ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [136ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [136ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [451ms] refreshing cache from master, [451ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [453ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [454ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [456ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [457ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [850ms] refreshing cache from master, [850ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [853ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@235864756(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@26d3d257 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873) -> (continuation of Deferred@408599516 after retry RPC@1331091850) -> (continuation of Deferred@1407144470 after retry RPC@708598638), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@732b94f3 -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873) -> (continuation of Deferred@408599516 after retry RPC@1331091850) -> (continuation of Deferred@1407144470 after retry RPC@708598638))) after lookup
18:51:33.957 [DEBUG - kudu-nio-0] (Connection.java:748) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] connecting to peer
18:51:33.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] cleaning up while in state CONNECTING due to: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:33.958 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345
18:51:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.450264 31231 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader fafc2bbe61c8493aa2bc1bfa367b392b)
18:51:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.450367 31231 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.450552 31231 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:34.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:34.451419 30883 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.454833 31154 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" is_pre_election: true
18:51:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.454981 31154 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 8.
18:51:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.455212 30882 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: fafc2bbe61c8493aa2bc1bfa367b392b
18:51:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.455360 31231 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 9 FOLLOWER]: Leader pre-election won for term 10
18:51:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.455438 31231 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 9 FOLLOWER]: Starting leader election (detected failure of leader fafc2bbe61c8493aa2bc1bfa367b392b)
18:51:34.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.455454 31231 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 9 FOLLOWER]: Advancing to term 10
18:51:34.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.456127 31231 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:34.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.456282 31231 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 10 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129:43067)
18:51:34.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.456570 31154 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "f8e2085855944ef6829cc9ccffc14be4" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "2dc38ce1e08949da90122cd3ab2b4a31"
18:51:34.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.456691 31154 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 8 FOLLOWER]: Advancing to term 10
18:51:34.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:34.456948 30883 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111)
18:51:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.458036 31154 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f8e2085855944ef6829cc9ccffc14be4 in term 10.
18:51:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.458276 30882 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, f8e2085855944ef6829cc9ccffc14be4; no voters: fafc2bbe61c8493aa2bc1bfa367b392b
18:51:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.458410 31231 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 10 FOLLOWER]: Leader election won for term 10
18:51:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.458600 31231 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 10 LEADER]: Becoming Leader. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Running, Role: LEADER
18:51:34.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.458715 31231 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:34.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.459632 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 reported cstate change: term changed from 9 to 10, leader changed from fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130) to f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131). New cstate: current_term: 10 leader_uuid: "f8e2085855944ef6829cc9ccffc14be4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: UNKNOWN } } }
18:51:34.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.937321 31154 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 10 FOLLOWER]: Refusing update from remote peer f8e2085855944ef6829cc9ccffc14be4: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
18:51:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:34.937901 31231 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
18:51:34.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:34.941464 30883 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 -> Peer fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345): Couldn't send request to peer fafc2bbe61c8493aa2bc1bfa367b392b. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.130:46345: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:35.635 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:35.635 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:35.636 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2533), Traces: [0ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [2ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [12ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [13ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [15ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [15ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [33ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [35ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [36ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [36ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [73ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [74ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [76ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [76ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [133ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [134ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [136ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [136ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [451ms] refreshing cache from master, [451ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [453ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [454ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [456ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [457ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [850ms] refreshing cache from master, [850ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [853ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [854ms] sending RPC to server fafc2bbe61c8493aa2bc1bfa367b392b, [856ms] received response from server fafc2bbe61c8493aa2bc1bfa367b392b: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [856ms] delaying RPC due to: Network error: Failed to connect to peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345): Connection refused: /127.28.7.130:46345, [2530ms] refreshing cache from master, [2530ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [2532ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@235864756(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@26d3d257 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873) -> (continuation of Deferred@408599516 after retry RPC@1331091850) -> (continuation of Deferred@1407144470 after retry RPC@708598638) -> (continuation of Deferred@2047183515 after retry RPC@2117634129), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@732b94f3 -> wakeup thread Time-limited test -> (continuation of Deferred@1926094829 after retry RPC@1567510859) -> (continuation of Deferred@1993522271 after retry RPC@1291198061) -> (continuation of Deferred@562497738 after retry RPC@1576955873) -> (continuation of Deferred@408599516 after retry RPC@1331091850) -> (continuation of Deferred@1407144470 after retry RPC@708598638) -> (continuation of Deferred@2047183515 after retry RPC@2117634129))) after lookup
18:51:35.636 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:35.638 [DEBUG - kudu-nio-1] (Connection.java:757) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] Successfully connected to peer
18:51:35.654 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf2c9bc7a, L:/127.0.0.1:36786 - R:/127.28.7.131:33291] using TOKEN/null
18:51:35.664 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.130:46345
18:51:35.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.665174 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:46345
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45681
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.759232 31273 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.759464 31273 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.759488 31273 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.760687 31273 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.760741 31273 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.130
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.762152 31273 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.130:46345
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.130
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45681
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31273
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.130
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/logs
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.762359 31273 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.762617 31273 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.767834 31279 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.767846 31281 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:35.767846 31278 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.768151 31273 server_base.cc:1034] running on GCE node
18:51:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.768407 31273 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.768712 31273 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.769893 31273 hybrid_clock.cc:648] HybridClock initialized: now 1786474295769876 us; error 29 us; skew 500 ppm
18:51:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.771333 31273 webserver.cc:460] Webserver started at http://127.28.7.130:45681/ using document root <none> and password file <none>
18:51:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.771616 31273 fs_manager.cc:359] Metadata directory not provided
18:51:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.771708 31273 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.773093 31273 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.001s	sys 0.000s
18:51:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.773850 31289 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.774066 31273 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.001s	sys 0.000s
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.774173 31273 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.774549 31273 fs_report.cc:383] FS layout report
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/wal
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/data
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.793081 31273 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.793500 31273 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.793681 31273 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:35.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.794001 31273 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:35.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.794531 31296 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:35.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.795845 31273 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.795917 31273 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.795956 31273 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.796707 31273 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.796763 31273 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:35.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.796801 31296 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap starting.
18:51:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.804453 31273 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.130:46345
18:51:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.804564 31426 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.130:46345 every 8 connection(s)
18:51:35.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.804950 31273 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-1/data/info.pb
18:51:35.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.805493 31296 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Log is configured to *not* fsync() on all Append() calls
18:51:35.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.810571 31296 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.810952 31296 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Bootstrap complete.
18:51:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.811666 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 31273
18:51:35.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:35.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.812177 31296 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent bootstrapping tablet: real 0.015s	user 0.002s	sys 0.010s
18:51:35.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.812686 31427 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:35.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.812786 31427 heartbeater.cc:464] Registering TS with master...
18:51:35.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.812990 31427 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:35.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.813601 28929 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.813359 31428 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.814069 31428 heartbeater.cc:464] Registering TS with master...
18:51:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.814152 31428 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.813593 31296 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.814296 31296 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: fafc2bbe61c8493aa2bc1bfa367b392b, State: Initialized, Role: FOLLOWER
18:51:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.814456 31296 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.814791 31296 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b: Time spent starting tablet: real 0.003s	user 0.004s	sys 0.001s
18:51:35.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.815222 28753 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:35.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.815624 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.130:45101
18:51:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.815936 31429 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.815984 31429 heartbeater.cc:464] Registering TS with master...
18:51:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.816179 31429 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.816561 28839 ts_manager.cc:194] Re-registered known tserver with Master: fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345)
18:51:35.816 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.817116 31428 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:35.817 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.28.7.131:33291
18:51:35.817 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:35.817759 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 30860
18:51:35.831 [DEBUG - kudu-nio-1] (Connection.java:239) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] handling channelInactive
18:51:35.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:35.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state READY due to: connection closed
18:51:35.831 [DEBUG - Time-limited test] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:35.833 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.833 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.834 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.855 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:35.855 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:35.856 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [3ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [23ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1420329208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@596ad11e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a97ba76 -> wakeup thread Time-limited test)) after lookup
18:51:35.856 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:35.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.859 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.875 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:35.875 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:35.875 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [3ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [23ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [25ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [28ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [28ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1420329208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@596ad11e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a97ba76 -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009))) after lookup
18:51:35.876 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:35.876 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.876 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.877 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.895 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:35.895 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:35.896 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=64), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [3ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [23ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [25ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [28ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [28ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [46ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [63ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1420329208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@596ad11e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a97ba76 -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825))) after lookup
18:51:35.897 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:35.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.898 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.995 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:35.996 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:35.996 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=164), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [3ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [23ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [25ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [28ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [28ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [46ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [63ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [65ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [67ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [67ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [162ms] refreshing cache from master, [162ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [164ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1420329208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@596ad11e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a97ba76 -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361))) after lookup
18:51:35.997 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:35.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.997 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:35.998 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:36.215 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:36.215 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:36.216 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=384), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [3ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [23ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [25ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [28ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [28ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [46ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [63ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [65ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [67ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [67ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [162ms] refreshing cache from master, [162ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [164ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [165ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [167ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [167ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [381ms] refreshing cache from master, [381ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [383ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1420329208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@596ad11e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361) -> (continuation of Deferred@2052533070 after retry RPC@90852704), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a97ba76 -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361) -> (continuation of Deferred@2052533070 after retry RPC@90852704))) after lookup
18:51:36.216 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:36.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:36.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:36.218 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:36.555 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:36.555 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067),f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)[L],fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:36.556 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=724), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [3ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [23ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [25ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [28ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [28ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [46ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [63ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [65ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [67ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [67ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [162ms] refreshing cache from master, [162ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [164ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [165ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [167ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [167ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [381ms] refreshing cache from master, [381ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [383ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [385ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [387ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [387ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [722ms] refreshing cache from master, [722ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [724ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1420329208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@596ad11e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361) -> (continuation of Deferred@2052533070 after retry RPC@90852704) -> (continuation of Deferred@294179912 after retry RPC@1188993620), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a97ba76 -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361) -> (continuation of Deferred@2052533070 after retry RPC@90852704) -> (continuation of Deferred@294179912 after retry RPC@1188993620))) after lookup
18:51:36.557 [DEBUG - kudu-nio-0] (Connection.java:748) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] connecting to peer
18:51:36.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:36.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] cleaning up while in state CONNECTING due to: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:36.558 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291) for tablet 1a1e672af9074011b1eba5773e6d0f4a: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291
18:51:37.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.164579 31444 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:37.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.164690 31444 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:37.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.165107 31444 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:37.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:37.165601 31078 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:37.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:37.166100 31078 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:37.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.168998 31367 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" is_pre_election: true
18:51:37.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.169185 31367 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 9.
18:51:37.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.169450 31081 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:37.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.169636 31444 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 10 FOLLOWER]: Leader pre-election won for term 11
18:51:37.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.169709 31444 raft_consensus.cc:482] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 10 FOLLOWER]: Starting leader election (detected failure of leader f8e2085855944ef6829cc9ccffc14be4)
18:51:37.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.169739 31444 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 10 FOLLOWER]: Advancing to term 11
18:51:37.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.170403 31444 raft_consensus.cc:504] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.170571 31444 leader_election.cc:288] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 11 election: Requested vote from peers fafc2bbe61c8493aa2bc1bfa367b392b (127.28.7.130:46345), f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.170799 31367 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a1e672af9074011b1eba5773e6d0f4a" candidate_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b"
18:51:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.170912 31367 raft_consensus.cc:3037] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 9 FOLLOWER]: Advancing to term 11
18:51:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:37.171178 31078 leader_election.cc:334] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111)
18:51:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.172173 31367 raft_consensus.cc:2454] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2dc38ce1e08949da90122cd3ab2b4a31 in term 11.
18:51:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.172407 31081 leader_election.cc:302] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2dc38ce1e08949da90122cd3ab2b4a31, fafc2bbe61c8493aa2bc1bfa367b392b; no voters: f8e2085855944ef6829cc9ccffc14be4
18:51:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.172561 31444 raft_consensus.cc:2781] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 11 FOLLOWER]: Leader election won for term 11
18:51:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.172739 31444 raft_consensus.cc:686] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [term 11 LEADER]: Becoming Leader. State: Replica: 2dc38ce1e08949da90122cd3ab2b4a31, State: Running, Role: LEADER
18:51:37.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.172914 31444 consensus_queue.cc:237] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.173853 28753 catalog_manager.cc:5495] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 reported cstate change: term changed from 10 to 11, leader changed from f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131) to 2dc38ce1e08949da90122cd3ab2b4a31 (127.28.7.129). New cstate: current_term: 11 leader_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } health_report { overall_health: HEALTHY } } }
18:51:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.697012 31367 raft_consensus.cc:1261] T 1a1e672af9074011b1eba5773e6d0f4a P fafc2bbe61c8493aa2bc1bfa367b392b [term 11 FOLLOWER]: Refusing update from remote peer 2dc38ce1e08949da90122cd3ab2b4a31: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
18:51:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.697424 31466 consensus_queue.cc:1035] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
18:51:37.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:37.701095 31078 consensus_peers.cc:487] T 1a1e672af9074011b1eba5773e6d0f4a P 2dc38ce1e08949da90122cd3ab2b4a31 -> Peer f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291): Couldn't send request to peer f8e2085855944ef6829cc9ccffc14be4. Status: Network error: Client connection negotiation failed: client connection to 127.28.7.131:33291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:51:37.935 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a1e672af9074011b1eba5773e6d0f4a for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680' with partition [<start>, <end>)
18:51:37.936 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1a1e672af9074011b1eba5773e6d0f4a@[2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)[L],f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291),fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)]}]
18:51:37.936 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1a1e672af9074011b1eba5773e6d0f4a, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2104), Traces: [0ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [3ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [3ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [21ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [23ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [25ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [28ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [28ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [43ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [44ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [46ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [46ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [63ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [65ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [67ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [67ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [162ms] refreshing cache from master, [162ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [164ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [165ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [167ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [167ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [381ms] refreshing cache from master, [381ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [383ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [385ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [387ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [387ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [722ms] refreshing cache from master, [722ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [724ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, [725ms] sending RPC to server f8e2085855944ef6829cc9ccffc14be4, [727ms] received response from server f8e2085855944ef6829cc9ccffc14be4: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [727ms] delaying RPC due to: Network error: Failed to connect to peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291): Connection refused: /127.28.7.131:33291, [2102ms] refreshing cache from master, [2102ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.7.190:38569, [2104ms] Sub RPC GetTableLocations: received response from server master-127.28.7.190:38569: OK, deferred=Deferred@1420329208(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$82/966761625@596ad11e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361) -> (continuation of Deferred@2052533070 after retry RPC@90852704) -> (continuation of Deferred@294179912 after retry RPC@1188993620) -> (continuation of Deferred@781817946 after retry RPC@104991283), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3a97ba76 -> wakeup thread Time-limited test -> (continuation of Deferred@2131601068 after retry RPC@2018441009) -> (continuation of Deferred@175020480 after retry RPC@791825) -> (continuation of Deferred@900092897 after retry RPC@1440902361) -> (continuation of Deferred@2052533070 after retry RPC@90852704) -> (continuation of Deferred@294179912 after retry RPC@1188993620) -> (continuation of Deferred@781817946 after retry RPC@104991283))) after lookup
18:51:37.937 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] connecting to peer
18:51:37.939 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] Successfully connected to peer
18:51:37.955 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x22fd37b5, L:/127.0.0.1:34668 - R:/127.28.7.129:43067] using TOKEN/null
18:51:37.964 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.28.7.131:33291
18:51:37.964 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:37.964898 28702 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIxmF87/build/release/bin/kudu
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:51:38.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.053066 31484 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:51:38.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.053289 31484 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:51:38.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.053321 31484 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:51:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.054570 31484 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:51:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.054628 31484 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.7.131
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.055938 31484 tablet_server_runner.cc:78] Tablet server non-default flags:
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.7.148:37585
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.7.131:33291
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.7.131
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33917
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.7.190:38569,127.28.7.189:39853,127.28.7.188:39825
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.31484
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.7.131
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/logs
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 17:59:43 UTC on ccycloud.quasar-xuakos.root.comops.site
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581133
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.056152 31484 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:51:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.056427 31484 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:51:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.061944 31490 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.061944 31493 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:51:38.062060 31491 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:51:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.062103 31484 server_base.cc:1034] running on GCE node
18:51:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.062678 31484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:51:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.063058 31484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:51:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.064258 31484 hybrid_clock.cc:648] HybridClock initialized: now 1786474298064243 us; error 30 us; skew 500 ppm
18:51:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.069841 31484 webserver.cc:460] Webserver started at http://127.28.7.131:33917/ using document root <none> and password file <none>
18:51:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.070207 31484 fs_manager.cc:359] Metadata directory not provided
18:51:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.070350 31484 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:51:38.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.072091 31484 fs_manager.cc:709] Time spent opening directory manager: real 0.001s	user 0.000s	sys 0.002s
18:51:38.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.073168 31499 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:51:38.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.073441 31484 fs_manager.cc:725] Time spent opening block manager: real 0.001s	user 0.000s	sys 0.001s
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.073580 31484 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data,/tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8e2085855944ef6829cc9ccffc14be4"
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:51:12 on dist-test-slave-77v9"
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.074007 31484 fs_report.cc:383] FS layout report
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/wal
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/data
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:51:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:51:38.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.088042 31484 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips  26 Jan 2017
18:51:38.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.088503 31484 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:51:38.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.088711 31484 kserver.cc:163] Server-wide thread pool size limit: 3276
18:51:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.089084 31484 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:51:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.089666 31506 ts_tablet_manager.cc:536] Loading tablet metadata (0/1 complete)
18:51:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.091190 31484 ts_tablet_manager.cc:579] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:51:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.091297 31484 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s	user 0.000s	sys 0.000s
18:51:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.091341 31484 ts_tablet_manager.cc:594] Registering tablets (0/1 complete)
18:51:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.092065 31484 ts_tablet_manager.cc:610] Registered 1 tablets
18:51:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.092128 31484 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s	user 0.000s	sys 0.000s
18:51:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.092209 31506 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap starting.
18:51:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.101081 31484 rpc_server.cc:307] RPC server started. Bound to: 127.28.7.131:33291
18:51:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.101465 31506 log.cc:824] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Log is configured to *not* fsync() on all Append() calls
18:51:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.101632 31484 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIxmF87/test-tmp/mini-kudu-cluster5554939420260683778/ts-2/data/info.pb
18:51:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.101522 31631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.7.131:33291 every 8 connection(s)
18:51:38.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.108073 31506 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:51:38.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.108490 31506 tablet_bootstrap.cc:492] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Bootstrap complete.
18:51:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.109925 31506 ts_tablet_manager.cc:1397] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent bootstrapping tablet: real 0.018s	user 0.009s	sys 0.007s
18:51:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.110868 28702 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIxmF87/build/release/bin/kudu as pid 31484
18:51:38.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:51:38.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.111696 31506 raft_consensus.cc:348] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:38.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.112325 31506 raft_consensus.cc:729] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8e2085855944ef6829cc9ccffc14be4, State: Initialized, Role: FOLLOWER
18:51:38.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.112440 31635 heartbeater.cc:347] Connected to a master server at 127.28.7.189:39853
18:51:38.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.112501 31506 consensus_queue.cc:260] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fafc2bbe61c8493aa2bc1bfa367b392b" member_type: VOTER last_known_addr { host: "127.28.7.130" port: 46345 } } peers { permanent_uuid: "f8e2085855944ef6829cc9ccffc14be4" member_type: VOTER last_known_addr { host: "127.28.7.131" port: 33291 } } peers { permanent_uuid: "2dc38ce1e08949da90122cd3ab2b4a31" member_type: VOTER last_known_addr { host: "127.28.7.129" port: 43067 } }
18:51:38.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.112555 31635 heartbeater.cc:464] Registering TS with master...
18:51:38.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.112852 31635 heartbeater.cc:511] Master 127.28.7.189:39853 requested a full tablet report, sending...
18:51:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.113664 31634 heartbeater.cc:347] Connected to a master server at 127.28.7.190:38569
18:51:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.113773 31634 heartbeater.cc:464] Registering TS with master...
18:51:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.113850 31634 heartbeater.cc:511] Master 127.28.7.190:38569 requested a full tablet report, sending...
18:51:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.113960 28839 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.114244 28753 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.114774 28753 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.7.131:36249
18:51:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.114929 31506 ts_tablet_manager.cc:1428] T 1a1e672af9074011b1eba5773e6d0f4a P f8e2085855944ef6829cc9ccffc14be4: Time spent starting tablet: real 0.005s	user 0.002s	sys 0.001s
18:51:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.115604 31634 heartbeater.cc:503] Master 127.28.7.190:38569 was elected leader, sending a full tablet report...
18:51:38.115 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:38.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.117324 31633 heartbeater.cc:347] Connected to a master server at 127.28.7.188:39825
18:51:38.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.117393 31633 heartbeater.cc:464] Registering TS with master...
18:51:38.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.117443 31633 heartbeater.cc:511] Master 127.28.7.188:39825 requested a full tablet report, sending...
18:51:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.117800 28923 ts_manager.cc:194] Re-registered known tserver with Master: f8e2085855944ef6829cc9ccffc14be4 (127.28.7.131:33291)
18:51:38.125 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1374) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786474272680, tablet=1a1e672af9074011b1eba5773e6d0f4a, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:51:38.131 [DEBUG - main] (Connection.java:575) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] disconnecting while in state READY
18:51:38.133 [DEBUG - main] (Connection.java:575) [peer master-127.28.7.188:39825(127.28.7.188:39825)] disconnecting while in state READY
18:51:38.133 [DEBUG - main] (Connection.java:575) [peer f8e2085855944ef6829cc9ccffc14be4(127.28.7.131:33291)] disconnecting while in state TERMINATED
18:51:38.134 [DEBUG - main] (Connection.java:575) [peer master-127.28.7.189:39853(127.28.7.189:39853)] disconnecting while in state READY
18:51:38.135 [DEBUG - main] (Connection.java:575) [peer fafc2bbe61c8493aa2bc1bfa367b392b(127.28.7.130:46345)] disconnecting while in state TERMINATED
18:51:38.136 [DEBUG - main] (Connection.java:575) [peer master-127.28.7.190:38569(127.28.7.190:38569)] disconnecting while in state READY
18:51:38.139 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.7.189:39853(127.28.7.189:39853)] handling channelInactive
18:51:38.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.7.189:39853(127.28.7.189:39853)] cleaning up while in state READY due to: connection closed
18:51:38.139 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.7.190:38569(127.28.7.190:38569)] handling channelInactive
18:51:38.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.7.190:38569(127.28.7.190:38569)] cleaning up while in state READY due to: connection closed
18:51:38.139 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:51:38.140 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] handling channelInactive
18:51:38.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2dc38ce1e08949da90122cd3ab2b4a31(127.28.7.129:43067)] cleaning up while in state READY due to: connection closed
18:51:38.141 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.7.188:39825(127.28.7.188:39825)] handling channelInactive
18:51:38.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.7.188:39825(127.28.7.188:39825)] cleaning up while in state READY due to: connection closed
18:51:38.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.141891 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 31061
18:51:38.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.153909 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 31273
18:51:38.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.166064 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 31484
18:51:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.173261 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 28715
18:51:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.180950 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 28801
18:51:38.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:51:38.188328 28702 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIxmF87/build/release/bin/kudu with pid 28890
18:51:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T18:51:38Z chronyd exiting

Time: 28.36

OK (1 test)

