JUnit version 4.13.2
.20:02:55.469 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1786478575461
20:02:55.532 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
20:02:55.598 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:02:55.612 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskK9yTow/test-tmp
20:02:55.618 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:02:55.622 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskK9yTow/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:02:55.876 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

20:02:56.079 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:02:56.080 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

20:02:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.085878 10693 mini_oidc.cc:150] Starting JWKS server
20:02:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.090008 10693 webserver.cc:533] Webserver started at http://127.0.0.1:42947/ using document root <none> and password file <none>
20:02:56.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.090629 10693 mini_oidc.cc:186] Starting OIDC Discovery server
20:02:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.093559 10693 webserver.cc:533] Webserver started at http://127.0.0.1:34413/ using document root <none> and password file <none>
20:02:56.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:02:56Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:02:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:02:56Z Disabled control of system clock
20:02:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.167335 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:02:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:02:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/wal
20:02:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data
20:02:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:02:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:02:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:02:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/logs
20:02:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data/info.pb
20:02:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:02:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:02:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:02:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:02:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:02:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:02:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.126:45721
20:02:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.126
20:02:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:02:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:02:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:02:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42947/jwks.json with env {}
20:02:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.470289 10709 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:02:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.470945 10709 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:02:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.471482 10709 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:02:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.503149 10709 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:02:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.503432 10709 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:02:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.503705 10709 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:02:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.503908 10709 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:02:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.504091 10709 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:02:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.504227 10709 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42947/jwks.json
20:02:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.540433 10709 master_runner.cc:387] Master server non-default flags:
20:02:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/logs
20:02:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:02:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:02:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data
20:02:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/wal
20:02:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:02:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.126:45721
20:02:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:02:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42947/jwks.json
20:02:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data/info.pb
20:02:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.126
20:02:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:02:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:02:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:02:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:02:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:02:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:02:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:02:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:02:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:02:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:02:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:02:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:02:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.542032 10709 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:02:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:56.543833 10709 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:02:56.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.556039 10719 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:57.960387 10718 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 10709
20:02:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.289772 10718 kernel_stack_watchdog.cc:198] Thread 10709 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:02:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:02:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:02:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:02:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:02:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:56.556856 10720 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.290702 10709 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.734s	user 0.610s	sys 1.122s
20:02:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.291136 10709 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.735s	user 0.610s	sys 1.122s
20:02:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.293005 10722 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.295632 10721 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1734 milliseconds
20:02:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.295644 10709 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:02:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.297166 10709 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:02:58.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.299549 10709 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:02:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.300899 10709 hybrid_clock.cc:648] HybridClock initialized: now 1786478578300837 us; error 113 us; skew 500 ppm
20:02:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.301824 10709 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:02:58.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.308239 10709 webserver.cc:533] Webserver started at http://127.10.113.126:41981/ using document root <none> and password file <none>
20:02:58.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.309332 10709 fs_manager.cc:362] Metadata directory not provided
20:02:58.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.309576 10709 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:02:58.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.309978 10709 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:02:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.314651 10709 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data/instance:
20:02:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb8448e1a39d46748895942435158dad"
20:02:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:02:58 on dist-test-slave-04bb"
20:02:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.315936 10709 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/wal/instance:
20:02:58.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb8448e1a39d46748895942435158dad"
20:02:58.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:02:58 on dist-test-slave-04bb"
20:02:58.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.324340 10709 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.001s	sys 0.009s
20:02:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.330356 10732 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:02:58.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.331625 10709 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
20:02:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.332018 10709 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/wal
20:02:58.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb8448e1a39d46748895942435158dad"
20:02:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:02:58 on dist-test-slave-04bb"
20:02:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.332445 10709 fs_report.cc:389] FS layout report
20:02:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:02:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/wal
20:02:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/wal
20:02:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data/data
20:02:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:02:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:02:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:02:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:02:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:02:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:02:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:02:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:02:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:02:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:02:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:02:58.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.391265 10709 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:02:58.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.393067 10709 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:02:58.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.393606 10709 kserver.cc:163] Server-wide thread pool size limit: 3276
20:02:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.467310 10709 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.126:45721
20:02:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.467401 10812 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.126:45721 every 8 connection(s)
20:02:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.470438 10709 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/data/info.pb
20:02:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.476658 10815 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:02:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.478767 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 10709
20:02:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.479319 10693 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-0/wal/instance
20:02:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.484627 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:02:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:02:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/wal
20:02:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data
20:02:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:02:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:02:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:02:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/logs
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data/info.pb
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:02:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:02:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:02:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.125:33831
20:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.125
20:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:02:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:02:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42947/jwks.json with env {}
20:02:58.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.493584 10815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } has no permanent_uuid. Determining permanent_uuid...
20:02:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.519567 10815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } has no permanent_uuid. Determining permanent_uuid...
20:02:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.522126 10734 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:02:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.526582 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.125:33831: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:02:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.576124 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } attempt: 1
20:02:58.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.581574 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.125:33831: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:02:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.649971 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } attempt: 2
20:02:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.655088 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.125:33831: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:02:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.746651 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } attempt: 3
20:02:58.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.750964 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.125:33831: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:02:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.803675 10819 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:02:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.804342 10819 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:02:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.805126 10819 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:02:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.848399 10819 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:02:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.848788 10819 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:02:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.849009 10819 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:02:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.849205 10819 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:02:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.849512 10819 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:02:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.849668 10819 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42947/jwks.json
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.887261 10819 master_runner.cc:387] Master server non-default flags:
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/logs
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data
20:02:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/wal
20:02:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:02:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.125:33831
20:02:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:02:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42947/jwks.json
20:02:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data/info.pb
20:02:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.125
20:02:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:02:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:02:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:02:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:02:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:02:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:02:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:02:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:02:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:02:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:02:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:02:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:02:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.888849 10819 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:02:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.890816 10819 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:02:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:58.894407 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } attempt: 4
20:02:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.900911 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.125:33831: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:02:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.904894 10833 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:59.200444 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } attempt: 5
20:02:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:59.211465 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.125:33831: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:02:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:59.759042 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } attempt: 6
20:02:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:59.801076 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.125:33831: Network error: Client connection negotiation failed: client connection to 127.10.113.125:33831: connect: Connection refused (error 111)
20:03:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.309585 10832 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 10819
20:03:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.426277 10832 kernel_stack_watchdog.cc:198] Thread 10819 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
20:03:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.426798 10819 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.520s	user 0.545s	sys 0.975s
20:03:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.427237 10819 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.521s	user 0.545s	sys 0.975s
20:03:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:58.907507 10835 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.429026 10846 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.429095 10819 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.429538 10845 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1519 milliseconds
20:03:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.434927 10819 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:00.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.437583 10819 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.438968 10819 hybrid_clock.cc:648] HybridClock initialized: now 1786478580438901 us; error 72 us; skew 500 ppm
20:03:00.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.439879 10819 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.446318 10819 webserver.cc:533] Webserver started at http://127.10.113.125:40841/ using document root <none> and password file <none>
20:03:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.447425 10819 fs_manager.cc:362] Metadata directory not provided
20:03:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.447674 10819 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.448271 10819 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.452706 10819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data/instance:
20:03:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28be70ce1eb74763ab63d482b89bef6b"
20:03:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:00 on dist-test-slave-04bb"
20:03:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.454037 10819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/wal/instance:
20:03:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28be70ce1eb74763ab63d482b89bef6b"
20:03:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:00 on dist-test-slave-04bb"
20:03:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.461655 10819 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.002s
20:03:00.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.468500 10862 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:00.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.469931 10819 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.001s
20:03:00.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.470350 10819 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/wal
20:03:00.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28be70ce1eb74763ab63d482b89bef6b"
20:03:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:00 on dist-test-slave-04bb"
20:03:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.470788 10819 fs_report.cc:389] FS layout report
20:03:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/wal
20:03:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/wal
20:03:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data/data
20:03:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.530117 10819 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.531849 10819 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.532276 10819 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:00.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.607939 10819 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.125:33831
20:03:00.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.608022 10936 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.125:33831 every 8 connection(s)
20:03:00.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.611627 10819 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/data/info.pb
20:03:00.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.617563 10939 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.620735 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 10819
20:03:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.621215 10693 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-1/wal/instance
20:03:00.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.624205 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:00.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/wal
20:03:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data
20:03:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/logs
20:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data/info.pb
20:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.124:42269
20:03:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.124
20:03:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42947/jwks.json with env {}
20:03:00.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.633358 10939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } has no permanent_uuid. Determining permanent_uuid...
20:03:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.661553 10939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } has no permanent_uuid. Determining permanent_uuid...
20:03:00.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.677505 10939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } has no permanent_uuid. Determining permanent_uuid...
20:03:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.680007 10866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:00.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.684867 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.734418 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 1
20:03:00.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.738826 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:00.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.807226 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 2
20:03:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.812069 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.861654 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } attempt: 7
20:03:00.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.874821 10815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } has no permanent_uuid. Determining permanent_uuid...
20:03:00.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.881325 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.903472 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 3
20:03:00.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.907897 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:00.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:00.939742 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 1
20:03:00.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.944034 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.961225 10942 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:00.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.961861 10942 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:00.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.962644 10942 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.996055 10942 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.996356 10942 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:00.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.996573 10942 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:00.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.996770 10942 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:00.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.996950 10942 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:00.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:00.997205 10942 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42947/jwks.json
20:03:01.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.025467 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 2
20:03:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.029894 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.034770 10942 master_runner.cc:387] Master server non-default flags:
20:03:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/logs
20:03:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data
20:03:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/wal
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.124:42269
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42947/jwks.json
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data/info.pb
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.124
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.036268 10942 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.038214 10942 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:01.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.049847 10964 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:01.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.051277 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 4
20:03:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.058997 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.115366 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 3
20:03:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.123929 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:01.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.264701 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 4
20:03:01.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.273838 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.358513 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 5
20:03:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.367185 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:01.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.557423 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 5
20:03:01.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.566315 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:01.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:01.914741 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 6
20:03:01.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.949254 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:02.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.118886 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 6
20:03:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:02.146615 10815 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:02.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:02.453233 10963 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 10942
20:03:02.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:01.050647 10965 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:02.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:02.909219 10942 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.859s	user 0.613s	sys 1.232s
20:03:02.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:02.910967 10942 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.861s	user 0.613s	sys 1.233s
20:03:02.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:02.912142 10967 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:02.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:02.914989 10966 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1859 milliseconds
20:03:02.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.915048 10942 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:02.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.916348 10942 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:02.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.918969 10942 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.920423 10942 hybrid_clock.cc:648] HybridClock initialized: now 1786478582920376 us; error 56 us; skew 500 ppm
20:03:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.921375 10942 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:02.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.927668 10942 webserver.cc:533] Webserver started at http://127.10.113.124:45471/ using document root <none> and password file <none>
20:03:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.928819 10942 fs_manager.cc:362] Metadata directory not provided
20:03:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.929085 10942 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.929492 10942 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.933714 10942 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data/instance:
20:03:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "311eeef7de044622a33426d2e8409118"
20:03:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:02 on dist-test-slave-04bb"
20:03:02.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.934752 10942 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/wal/instance:
20:03:02.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "311eeef7de044622a33426d2e8409118"
20:03:02.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:02 on dist-test-slave-04bb"
20:03:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.942066 10942 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.006s
20:03:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.947896 10981 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.948994 10942 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
20:03:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.949369 10942 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/wal
20:03:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "311eeef7de044622a33426d2e8409118"
20:03:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:02 on dist-test-slave-04bb"
20:03:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:02.949761 10942 fs_report.cc:389] FS layout report
20:03:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/wal
20:03:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/wal
20:03:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data/data
20:03:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:03.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.009876 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 7
20:03:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:03.016921 10939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.113.124:42269: Network error: Client connection negotiation failed: client connection to 127.10.113.124:42269: connect: Connection refused (error 111)
20:03:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.017488 10942 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:03.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.018952 10942 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:03.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.019686 10942 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.095742 10942 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.124:42269
20:03:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.095856 11073 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.124:42269 every 8 connection(s)
20:03:03.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.098781 10942 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/data/info.pb
20:03:03.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.102959 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 10942
20:03:03.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.103494 10693 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/master-2/wal/instance
20:03:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.106506 11074 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:03.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.126543 11074 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } has no permanent_uuid. Determining permanent_uuid...
20:03:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.151103 11074 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } has no permanent_uuid. Determining permanent_uuid...
20:03:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.161026 11074 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } has no permanent_uuid. Determining permanent_uuid...
20:03:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.177671 11074 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118: Bootstrap starting.
20:03:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.180168 10815 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 7
20:03:03.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.187165 11074 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118: Neither blocks nor log segments found. Creating new log.
20:03:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.189368 11074 log.cc:826] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118: Log is configured to *not* fsync() on all Append() calls
20:03:03.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.200204 11074 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118: No bootstrap required, opened a new log
20:03:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.203451 10815 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: Bootstrap starting.
20:03:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.210116 10815 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: Neither blocks nor log segments found. Creating new log.
20:03:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.212890 10815 log.cc:826] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: Log is configured to *not* fsync() on all Append() calls
20:03:03.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.217890 10815 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: No bootstrap required, opened a new log
20:03:03.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.223822 11074 raft_consensus.cc:359] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:03.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.224670 11074 raft_consensus.cc:385] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:03.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.224892 11074 raft_consensus.cc:740] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 311eeef7de044622a33426d2e8409118, State: Initialized, Role: FOLLOWER
20:03:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.225755 11074 consensus_queue.cc:260] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:03.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.228953 11090 sys_catalog.cc:455] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:03.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.229892 11090 sys_catalog.cc:458] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: This master's current role is: FOLLOWER
20:03:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.230863 11074 sys_catalog.cc:565] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: configured and running, proceeding with master startup.
20:03:03.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.241698 10815 raft_consensus.cc:359] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.242709 10815 raft_consensus.cc:385] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.242937 10815 raft_consensus.cc:740] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fb8448e1a39d46748895942435158dad, State: Initialized, Role: FOLLOWER
20:03:03.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.243701 10815 consensus_queue.cc:260] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:03.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.247015 11100 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:03.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.248469 11100 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [sys.catalog]: This master's current role is: FOLLOWER
20:03:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.250001 10815 sys_catalog.cc:565] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [sys.catalog]: configured and running, proceeding with master startup.
20:03:03.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:03.275337 10997 tablet.cc:2406] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:03.279488 11113 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:03.281584 11113 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:03.315552 11123 catalog_manager.cc:1594] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:03.316509 11123 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:03.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.266963 10942 master_runner.cc:187] Error getting master registration for 127.10.113.126:45721: OK, instance_id { permanent_uuid: "fb8448e1a39d46748895942435158dad" instance_seqno: 1786478578447937 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:03.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.320349 10942 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.315057 10709 master_runner.cc:187] Error getting master registration for 127.10.113.125:33831: OK, instance_id { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" instance_seqno: 1786478580587666 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.341842 10709 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:03.404775 10741 tablet.cc:2406] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.442286 11090 raft_consensus.cc:493] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.442955 11090 raft_consensus.cc:515] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.447255 10778 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "311eeef7de044622a33426d2e8409118" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fb8448e1a39d46748895942435158dad" is_pre_election: true
20:03:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.448207 10778 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 311eeef7de044622a33426d2e8409118 in term 0.
20:03:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.451579 10984 leader_election.cc:304] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 311eeef7de044622a33426d2e8409118, fb8448e1a39d46748895942435158dad; no voters: 
20:03:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.450613 11090 leader_election.cc:290] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fb8448e1a39d46748895942435158dad (127.10.113.126:45721), 28be70ce1eb74763ab63d482b89bef6b (127.10.113.125:33831)
20:03:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.454790 11090 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.455726 11090 raft_consensus.cc:493] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:03.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.456636 11090 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 0 FOLLOWER]: Advancing to term 1
20:03:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.466982 11090 raft_consensus.cc:515] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:03.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.470880 10778 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "311eeef7de044622a33426d2e8409118" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fb8448e1a39d46748895942435158dad"
20:03:03.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.471505 10778 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [term 0 FOLLOWER]: Advancing to term 1
20:03:03.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.471776 11090 leader_election.cc:290] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [CANDIDATE]: Term 1 election: Requested vote from peers fb8448e1a39d46748895942435158dad (127.10.113.126:45721), 28be70ce1eb74763ab63d482b89bef6b (127.10.113.125:33831)
20:03:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.479305 10778 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 311eeef7de044622a33426d2e8409118 in term 1.
20:03:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.497442 10984 leader_election.cc:304] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 311eeef7de044622a33426d2e8409118, fb8448e1a39d46748895942435158dad; no voters: 
20:03:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.498381 11090 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 1 FOLLOWER]: Leader election won for term 1
20:03:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.498903 11090 raft_consensus.cc:697] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [term 1 LEADER]: Becoming Leader. State: Replica: 311eeef7de044622a33426d2e8409118, State: Running, Role: LEADER
20:03:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.499889 11090 consensus_queue.cc:237] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.519630 11127 sys_catalog.cc:455] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 311eeef7de044622a33426d2e8409118. Latest consensus state: current_term: 1 leader_uuid: "311eeef7de044622a33426d2e8409118" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.520689 11127 sys_catalog.cc:458] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: This master's current role is: LEADER
20:03:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.549469 11131 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
20:03:03.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.558252 11131 catalog_manager.cc:1520] Initializing Kudu cluster ID...
20:03:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.601559 10778 raft_consensus.cc:1275] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [term 1 FOLLOWER]: Refusing update from remote peer 311eeef7de044622a33426d2e8409118: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.603442 11127 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.662293 11100 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [sys.catalog]: SysCatalogTable state changed. Reason: New leader 311eeef7de044622a33426d2e8409118. Latest consensus state: current_term: 1 leader_uuid: "311eeef7de044622a33426d2e8409118" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.663005 11100 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [sys.catalog]: This master's current role is: FOLLOWER
20:03:03.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.668650 11134 mvcc.cc:204] Tried to move back new op lower bound from 7317416278405804032 to 7317416278025711616. Current Snapshot: MvccSnapshot[applied={T|T < 7317416278405804032}]
20:03:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.675217 11127 sys_catalog.cc:455] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "311eeef7de044622a33426d2e8409118" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.675976 11127 sys_catalog.cc:458] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: This master's current role is: LEADER
20:03:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.678020 11131 catalog_manager.cc:1383] Generated new cluster ID: 677a5760573b497986345b1a25d37aef
20:03:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.678349 11131 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
20:03:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.682143 11100 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "311eeef7de044622a33426d2e8409118" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.682821 11100 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad [sys.catalog]: This master's current role is: FOLLOWER
20:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.740417 11131 catalog_manager.cc:1406] Generated new certificate authority record
20:03:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.743563 11131 catalog_manager.cc:1540] Loading token signing keys...
20:03:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.778939 11131 catalog_manager.cc:6092] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118: Generated new TSK 0
20:03:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.780553 11131 catalog_manager.cc:1550] Initializing in-progress tserver states...
20:03:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:03.798321 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:0
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:04.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:04.314380 11141 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:04.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:04.323174 11123 catalog_manager.cc:1295] Loaded cluster ID: 677a5760573b497986345b1a25d37aef
20:03:04.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:04.323995 11123 catalog_manager.cc:1588] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: loading cluster ID for follower catalog manager: success
20:03:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:04.328302 11141 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:04.329162 11141 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:04.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:04.344595 11123 catalog_manager.cc:1610] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: acquiring CA information for follower catalog manager: success
20:03:04.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:04.350967 11123 catalog_manager.cc:1638] T 00000000000000000000000000000000 P fb8448e1a39d46748895942435158dad: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:04.401355 11141 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:04.402741 11141 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.65
20:03:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:04.470810 11141 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:0
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:04.472658 11141 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:04.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:04.474787 11141 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:04.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:04.504326 11154 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.107329 10939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } attempt: 8
20:03:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.177939 10939 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b: Bootstrap starting.
20:03:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:05.182255 10982 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.10.113.125:33831 timed out after 1.710s (SENT)
20:03:05.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.188340 10939 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b: Neither blocks nor log segments found. Creating new log.
20:03:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.208274 10939 log.cc:826] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b: Log is configured to *not* fsync() on all Append() calls
20:03:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.228549 10939 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b: No bootstrap required, opened a new log
20:03:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.298012 10939 raft_consensus.cc:359] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.299589 10939 raft_consensus.cc:385] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:05.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.300252 10939 raft_consensus.cc:740] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 28be70ce1eb74763ab63d482b89bef6b, State: Initialized, Role: FOLLOWER
20:03:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.301851 10939 consensus_queue.cc:260] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } }
20:03:05.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.324016 10939 sys_catalog.cc:565] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [sys.catalog]: configured and running, proceeding with master startup.
20:03:05.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.323899 10897 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "311eeef7de044622a33426d2e8409118" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "28be70ce1eb74763ab63d482b89bef6b"
20:03:05.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.325807 10898 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [term 0 FOLLOWER]: Advancing to term 1
20:03:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.324669 10899 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "311eeef7de044622a33426d2e8409118" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "28be70ce1eb74763ab63d482b89bef6b"
20:03:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:05.328075 10899 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:46580 (request call id 2) took 1856 ms (client timeout 1710 ms). Trace:
20:03:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:05.328455 10899 rpcz_store.cc:269] 0811 20:03:03.471496 (+     0us) service_pool.cc:167] Inserting onto call queue
20:03:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:03.471658 (+   162us) service_pool.cc:224] Handling call
20:03:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:05.327989 (+1856331us) inbound_call.cc:177] Queueing success response
20:03:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:03:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.323935 10902 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "311eeef7de044622a33426d2e8409118" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "28be70ce1eb74763ab63d482b89bef6b" is_pre_election: true
20:03:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:05.330422 10902 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:46580 (request call id 1) took 1880 ms (client timeout 1944 ms). Trace:
20:03:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:05.330772 10902 rpcz_store.cc:269] 0811 20:03:03.449946 (+     0us) service_pool.cc:167] Inserting onto call queue
20:03:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:03.450102 (+   156us) service_pool.cc:224] Handling call
20:03:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:05.330338 (+1880236us) inbound_call.cc:177] Queueing success response
20:03:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":19840}
20:03:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.324981 11166 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.332692 11166 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [sys.catalog]: This master's current role is: FOLLOWER
20:03:05.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.338526 10898 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [term 1 FOLLOWER]: Refusing update from remote peer 311eeef7de044622a33426d2e8409118: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:05.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.339425 10898 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:46580 (request call id 3) took 1740 ms. Trace:
20:03:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.340303 10898 rpcz_store.cc:276] 0811 20:03:03.599441 (+     0us) service_pool.cc:167] Inserting onto call queue
20:03:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:03.599610 (+   169us) service_pool.cc:224] Handling call
20:03:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:05.325511 (+1725901us) raft_consensus.cc:1471] Updating replica for 0 ops
20:03:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:05.339048 (+ 13537us) raft_consensus.cc:1697] Filling consensus response to leader.
20:03:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:05.339363 (+   315us) inbound_call.cc:177] Queueing success response
20:03:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:03:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.382781 11171 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [LEADER]: Connected to new peer: Peer: permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:05.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.451104 11166 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 311eeef7de044622a33426d2e8409118. Latest consensus state: current_term: 1 leader_uuid: "311eeef7de044622a33426d2e8409118" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.453501 11183 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "311eeef7de044622a33426d2e8409118" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.456542 11183 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [sys.catalog]: This master's current role is: FOLLOWER
20:03:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.457582 11166 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b [sys.catalog]: This master's current role is: FOLLOWER
20:03:05.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.480286 11192 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:03:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.785017 11169 sys_catalog.cc:455] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "311eeef7de044622a33426d2e8409118" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fb8448e1a39d46748895942435158dad" member_type: VOTER last_known_addr { host: "127.10.113.126" port: 45721 } } peers { permanent_uuid: "28be70ce1eb74763ab63d482b89bef6b" member_type: VOTER last_known_addr { host: "127.10.113.125" port: 33831 } } peers { permanent_uuid: "311eeef7de044622a33426d2e8409118" member_type: VOTER last_known_addr { host: "127.10.113.124" port: 42269 } } }
20:03:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:05.785779 11169 sys_catalog.cc:458] T 00000000000000000000000000000000 P 311eeef7de044622a33426d2e8409118 [sys.catalog]: This master's current role is: LEADER
20:03:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.500852 11192 catalog_manager.cc:1295] Loaded cluster ID: 677a5760573b497986345b1a25d37aef
20:03:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.501185 11192 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b: loading cluster ID for follower catalog manager: success
20:03:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.516724 11192 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b: acquiring CA information for follower catalog manager: success
20:03:06.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.528146 11192 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 28be70ce1eb74763ab63d482b89bef6b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:05.898052 11152 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 11141
20:03:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:06.846288 11141 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.339s	user 0.647s	sys 1.250s
20:03:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:06.853974 11141 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.346s	user 0.647s	sys 1.251s
20:03:06.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:04.509464 11156 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:06.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.862264 11141 server_base.cc:1061] running on GCE node
20:03:06.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:06.863903 11203 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.865815 11141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.873217 11141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.874862 11141 hybrid_clock.cc:648] HybridClock initialized: now 1786478586874822 us; error 41 us; skew 500 ppm
20:03:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.876268 11141 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:06.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.890444 11141 webserver.cc:533] Webserver started at http://127.10.113.65:42403/ using document root <none> and password file <none>
20:03:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.891536 11141 fs_manager.cc:362] Metadata directory not provided
20:03:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.891815 11141 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.892364 11141 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.897087 11141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/instance:
20:03:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:06 on dist-test-slave-04bb"
20:03:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.898396 11141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal/instance:
20:03:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:06 on dist-test-slave-04bb"
20:03:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.906826 11141 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.006s	sys 0.003s
20:03:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.913393 11213 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:06.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.914803 11141 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.000s	sys 0.005s
20:03:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.915179 11141 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:06 on dist-test-slave-04bb"
20:03:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.915625 11141 fs_report.cc:389] FS layout report
20:03:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/data
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:06.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.994094 11141 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.995831 11141 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:06.996390 11141 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:07.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.000185 11141 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.005234 11141 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.005471 11141 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.005815 11141 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.006035 11141 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.235630 11141 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.65:46157
20:03:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.235801 11378 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.65:46157 every 8 connection(s)
20:03:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.239152 11141 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:07.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.242355 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 11141
20:03:07.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.243033 10693 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal/instance
20:03:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.275108 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:0
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.320904 11379 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.321453 11379 heartbeater.cc:461] Registering TS with master...
20:03:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.322865 11379 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.324119 11380 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.324469 11380 heartbeater.cc:461] Registering TS with master...
20:03:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.325366 11380 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.328218 11384 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.328578 11384 heartbeater.cc:461] Registering TS with master...
20:03:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.329497 11384 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.329478 11018 ts_manager.cc:194] Registered new tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.329627 10887 ts_manager.cc:194] Registered new tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.333024 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.65:47073
20:03:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.333240 10767 ts_manager.cc:194] Registered new tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:07.639261 11392 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:07.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:07.639809 11392 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:07.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:07.640421 11392 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:07.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:07.675087 11392 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:07.675889 11392 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.66
20:03:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.720173 11392 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:0
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.721947 11392 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:07.723932 11392 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:07.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:07.737159 11402 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:08.337903 11379 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:09.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:09.140415 11401 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 11392
20:03:09.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:09.616848 11405 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:07.738369 11403 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:09.614586 11392 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.876s	user 0.583s	sys 1.206s
20:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:09.619047 11392 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.881s	user 0.584s	sys 1.206s
20:03:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:09.621809 11404 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1883 milliseconds
20:03:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.621865 11392 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.623360 11392 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:09.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.628819 11392 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:09.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.630461 11392 hybrid_clock.cc:648] HybridClock initialized: now 1786478589630382 us; error 89 us; skew 500 ppm
20:03:09.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.631356 11392 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.639145 11392 webserver.cc:533] Webserver started at http://127.10.113.66:33205/ using document root <none> and password file <none>
20:03:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.640116 11392 fs_manager.cc:362] Metadata directory not provided
20:03:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.640378 11392 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.640949 11392 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.645535 11392 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/instance:
20:03:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:09 on dist-test-slave-04bb"
20:03:09.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.646754 11392 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal/instance:
20:03:09.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:09.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:09 on dist-test-slave-04bb"
20:03:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.654990 11392 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.000s	sys 0.009s
20:03:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.661242 11424 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.662529 11392 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
20:03:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.662904 11392 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:09 on dist-test-slave-04bb"
20:03:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.663277 11392 fs_report.cc:389] FS layout report
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/data
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.723574 11392 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:09.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.725281 11392 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.725811 11392 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.729331 11392 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.734026 11392 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.734269 11392 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.734603 11392 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.734822 11392 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.908990 11392 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.66:34377
20:03:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.909106 11610 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.66:34377 every 8 connection(s)
20:03:09.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.911775 11392 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.917778 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 11392
20:03:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.918254 10693 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal/instance
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.938864 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:0
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:09.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:09.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:09.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:09.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:09.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:09.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.962885 11611 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:09.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.963397 11611 heartbeater.cc:461] Registering TS with master...
20:03:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.964772 11611 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.967545 11613 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.967929 11613 heartbeater.cc:461] Registering TS with master...
20:03:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.968092 11018 ts_manager.cc:194] Registered new tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.968936 11613 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.969909 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.66:56135
20:03:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.970139 11612 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.970458 11612 heartbeater.cc:461] Registering TS with master...
20:03:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.971349 11612 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.972877 10767 ts_manager.cc:194] Registered new tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:09.974201 10887 ts_manager.cc:194] Registered new tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:10.279268 11626 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:10.279752 11626 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:10.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:10.280157 11626 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:10.314261 11626 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:10.315047 11626 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.67
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:10.352872 11626 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:10.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:0
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:10.354386 11626 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:10.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:10.355923 11626 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:10.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:10.370365 11635 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:10.974360 11611 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:11.773419 11632 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 11626
20:03:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:10.371189 11636 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:11.846156 11632 kernel_stack_watchdog.cc:198] Thread 11626 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
20:03:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:11.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:11.848660 11626 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.478s	user 0.496s	sys 0.947s
20:03:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:11.849114 11626 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.479s	user 0.496s	sys 0.947s
20:03:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:11.850358 11642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:11.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.857102 11626 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:11.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:11.857157 11638 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1476 milliseconds
20:03:11.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.858603 11626 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.861007 11626 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.862502 11626 hybrid_clock.cc:648] HybridClock initialized: now 1786478591862469 us; error 52 us; skew 500 ppm
20:03:11.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.863430 11626 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.870133 11626 webserver.cc:533] Webserver started at http://127.10.113.67:32785/ using document root <none> and password file <none>
20:03:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.871161 11626 fs_manager.cc:362] Metadata directory not provided
20:03:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.871424 11626 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.871977 11626 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.876348 11626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/instance:
20:03:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:11 on dist-test-slave-04bb"
20:03:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.877631 11626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal/instance:
20:03:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:11 on dist-test-slave-04bb"
20:03:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.885550 11626 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.002s
20:03:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.891465 11656 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.892692 11626 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
20:03:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.893075 11626 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:11 on dist-test-slave-04bb"
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.893424 11626 fs_report.cc:389] FS layout report
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/data
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.953332 11626 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.954944 11626 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.955482 11626 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.958514 11626 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.962777 11626 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.963019 11626 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.963358 11626 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:11.963598 11626 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.123457 11626 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.67:37831
20:03:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.123590 11843 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.67:37831 every 8 connection(s)
20:03:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.126271 11626 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.136875 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 11626
20:03:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.137518 10693 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal/instance
20:03:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.178685 11845 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.179234 11845 heartbeater.cc:461] Registering TS with master...
20:03:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.181305 11846 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.181692 11846 heartbeater.cc:461] Registering TS with master...
20:03:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.181308 11845 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.182962 11846 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.184252 11844 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.184660 11844 heartbeater.cc:461] Registering TS with master...
20:03:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.185567 11844 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.185848 10767 ts_manager.cc:194] Registered new tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.186599 10887 ts_manager.cc:194] Registered new tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.188707 11018 ts_manager.cc:194] Registered new tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.190418 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.67:59187
20:03:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:12.205739 10693 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:03:12.217 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:03:12.219 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

20:03:12.229 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.10.113.126"
      port: 45721
    }
    bound_http_address {
      host: "127.10.113.126"
      port: 41981
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.10.113.125"
      port: 33831
    }
    bound_http_address {
      host: "127.10.113.125"
      port: 40841
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.10.113.124"
      port: 42269
    }
    bound_http_address {
      host: "127.10.113.124"
      port: 45471
    }
  }
}

20:03:12.232 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

20:03:12.235 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.10.113.65"
      port: 46157
    }
    bound_http_address {
      host: "127.10.113.65"
      port: 42403
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.10.113.66"
      port: 34377
    }
    bound_http_address {
      host: "127.10.113.66"
      port: 33205
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.10.113.67"
      port: 37831
    }
    bound_http_address {
      host: "127.10.113.67"
      port: 32785
    }
  }
}

20:03:12.235 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:03:12.468 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:03:12.544 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
20:03:12.741 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.113.126:45721(127.10.113.126:45721)] connecting to peer
20:03:12.793 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.113.125:33831(127.10.113.125:33831)] connecting to peer
20:03:12.794 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.113.124:42269(127.10.113.124:42269)] connecting to peer
20:03:12.811 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.113.125:33831(127.10.113.125:33831)] Successfully connected to peer
20:03:12.811 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.10.113.124:42269(127.10.113.124:42269)] Successfully connected to peer
20:03:12.815 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.113.126:45721(127.10.113.126:45721)] Successfully connected to peer
20:03:12.912 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.10.113.125
20:03:12.912 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.10.113.126
20:03:12.913 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.10.113.124
20:03:13.148 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x5be794e3, L:/127.0.0.1:44866 - R:/127.10.113.126:45721] using SASL/PLAIN
20:03:13.150 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x7b5942b1, L:/127.0.0.1:46590 - R:/127.10.113.124:42269] using SASL/PLAIN
20:03:13.152 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xa2027af3, L:/127.0.0.1:51492 - R:/127.10.113.125:33831] using SASL/PLAIN
20:03:13.193 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:03:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.193636 11844 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:13.195 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=502), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.113.126:45721, [96ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.113.125:33831, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.113.124:42269, [471ms] Sub RPC ConnectToMaster: received response from server master-127.10.113.126:45721: OK, [473ms] Sub RPC ConnectToMaster: received response from server master-127.10.113.124:42269: OK, [475ms] Sub RPC ConnectToMaster: received response from server master-127.10.113.125:33831: OK, deferred=null) after lookup
20:03:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.248324 11018 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46590:
20:03:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674"
20:03:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:03:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
20:03:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:03:13.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:13.252699 11018 catalog_manager.cc:7099] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:03:13.339 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 2e4c77cc77564788b7c0cd6e7fb87a46
20:03:13.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.355783 11738 tablet_service.cc:1511] Processing CreateTablet for tablet 242ddce46e42481b9276d573742023bb (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674 [id=2e4c77cc77564788b7c0cd6e7fb87a46]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:13.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.357719 11738 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 242ddce46e42481b9276d573742023bb. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:13.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.370039 11288 tablet_service.cc:1511] Processing CreateTablet for tablet 242ddce46e42481b9276d573742023bb (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674 [id=2e4c77cc77564788b7c0cd6e7fb87a46]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.372099 11288 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 242ddce46e42481b9276d573742023bb. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:13.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.375154 11497 tablet_service.cc:1511] Processing CreateTablet for tablet 242ddce46e42481b9276d573742023bb (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674 [id=2e4c77cc77564788b7c0cd6e7fb87a46]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.377130 11497 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 242ddce46e42481b9276d573742023bb. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:13.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.406147 11895 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap starting.
20:03:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.417505 11897 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap starting.
20:03:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.418550 11895 tablet_bootstrap.cc:654] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Neither blocks nor log segments found. Creating new log.
20:03:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.418902 11900 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap starting.
20:03:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.421059 11895 log.cc:826] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Log is configured to *not* fsync() on all Append() calls
20:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.426438 11900 tablet_bootstrap.cc:654] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Neither blocks nor log segments found. Creating new log.
20:03:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.428902 11900 log.cc:826] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Log is configured to *not* fsync() on all Append() calls
20:03:13.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.431386 11897 tablet_bootstrap.cc:654] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Neither blocks nor log segments found. Creating new log.
20:03:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.434464 11897 log.cc:826] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Log is configured to *not* fsync() on all Append() calls
20:03:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.435101 11895 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: No bootstrap required, opened a new log
20:03:13.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.438462 11900 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: No bootstrap required, opened a new log
20:03:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.439002 11900 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent bootstrapping tablet: real 0.021s	user 0.007s	sys 0.009s
20:03:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.438144 11895 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent bootstrapping tablet: real 0.033s	user 0.016s	sys 0.009s
20:03:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.460407 11897 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: No bootstrap required, opened a new log
20:03:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.461043 11897 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent bootstrapping tablet: real 0.045s	user 0.013s	sys 0.022s
20:03:13.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.471865 11900 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.472860 11900 raft_consensus.cc:385] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:13.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.473170 11900 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Initialized, Role: FOLLOWER
20:03:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.473866 11895 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.474782 11895 raft_consensus.cc:385] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.475119 11895 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Initialized, Role: FOLLOWER
20:03:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.474898 11900 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.476286 11895 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.482998 11895 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent starting tablet: real 0.044s	user 0.039s	sys 0.000s
20:03:13.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.499644 11900 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent starting tablet: real 0.060s	user 0.027s	sys 0.030s
20:03:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.520609 11897 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.521397 11897 raft_consensus.cc:385] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.521703 11897 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6719b8a2cff04c88afd580790d9c389b, State: Initialized, Role: FOLLOWER
20:03:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.522791 11897 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.528106 11897 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent starting tablet: real 0.067s	user 0.032s	sys 0.005s
20:03:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:13.558182 11391 tablet.cc:2406] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.580470 11909 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.581125 11909 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.584602 11909 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:13.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.599390 11321 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b" is_pre_election: true
20:03:13.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.600427 11321 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 0.
20:03:13.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.601886 11530 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7" is_pre_election: true
20:03:13.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.602109 11658 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: 
20:03:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.602843 11530 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 0.
20:03:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.603042 11909 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.603420 11909 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:13.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.603749 11909 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 0 FOLLOWER]: Advancing to term 1
20:03:13.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.610641 11909 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.612574 11909 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 1 election: Requested vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.613462 11530 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.614001 11530 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 0 FOLLOWER]: Advancing to term 1
20:03:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.614319 11321 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.614866 11321 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 0 FOLLOWER]: Advancing to term 1
20:03:13.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.621272 11530 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 1.
20:03:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.623261 11660 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7; no voters: 
20:03:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.624399 11909 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 1 FOLLOWER]: Leader election won for term 1
20:03:13.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.625703 11321 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 1.
20:03:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.628428 11909 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 1 LEADER]: Becoming Leader. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Running, Role: LEADER
20:03:13.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.629683 11909 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.646579 11018 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 reported cstate change: term changed from 0 to 1, leader changed from <none> to 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67). New cstate: current_term: 1 leader_uuid: "12f7da6233f74ebd81a5cab80b3b3648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:03:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:13.650480 11854 tablet.cc:2406] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:13.698238 11619 tablet.cc:2406] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:13.782 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 2e4c77cc77564788b7c0cd6e7fb87a46
20:03:13.802 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:13.807 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:13.809 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [9ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=null) after lookup
20:03:13.819 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:13.822 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] Successfully connected to peer
20:03:13.898 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0xc0f439a5, L:/127.0.0.1:55268 - R:/127.10.113.67:37831] using TOKEN/null
20:03:13.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.913755 11321 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 1 FOLLOWER]: Refusing update from remote peer 12f7da6233f74ebd81a5cab80b3b3648: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:13.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.914149 11530 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 1 FOLLOWER]: Refusing update from remote peer 12f7da6233f74ebd81a5cab80b3b3648: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.915706 11909 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.916905 11916 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:13.941161 11924 mvcc.cc:204] Tried to move back new op lower bound from 7317416320652611584 to 7317416319518265344. Current Snapshot: MvccSnapshot[applied={T|T < 7317416320652611584}]
20:03:14.097 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:14.099 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.67:37831
20:03:14.103 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:14.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:14.104318 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 11626
20:03:14.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:14.155 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] handling channelInactive
20:03:14.156 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state READY due to: connection closed
20:03:14.174 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:14.177 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.177 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.177 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.198 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:14.198 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:14.199 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=3, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [21ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [21ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1413778483(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@e6098f2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a4fbb -> wakeup thread Time-limited test)) after lookup
20:03:14.199 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:14.202 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.202 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.202 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.218 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:14.218 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:14.218 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=5, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [21ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [21ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [43ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [46ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [46ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [61ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1413778483(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@e6098f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a4fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381))) after lookup
20:03:14.219 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:14.221 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.221 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.221 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.259 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:14.259 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:14.259 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=7, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=103), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [21ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [21ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [43ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [46ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [46ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [61ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [63ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [65ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [66ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [102ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1413778483(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@e6098f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a4fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622))) after lookup
20:03:14.260 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:14.262 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.262 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.262 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.499 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:14.499 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:14.500 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=9, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=344), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [21ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [21ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [43ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [46ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [46ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [61ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [63ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [65ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [66ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [102ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [104ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [106ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [106ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [337ms] refreshing cache from master, [338ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [342ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1413778483(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@e6098f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622) -> (continuation of Deferred@1137110874 after retry RPC@709474206), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a4fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622) -> (continuation of Deferred@1137110874 after retry RPC@709474206))) after lookup
20:03:14.501 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:14.503 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:14.503 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:15.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:15.418 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:15.419 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=11, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1263), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [21ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [21ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [43ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [46ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [46ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [61ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [63ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [65ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [66ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [102ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [104ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [106ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [106ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [337ms] refreshing cache from master, [338ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [342ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [345ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [347ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [347ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [1257ms] refreshing cache from master, [1258ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1262ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1413778483(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@e6098f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622) -> (continuation of Deferred@1137110874 after retry RPC@709474206) -> (continuation of Deferred@1864980661 after retry RPC@138147721), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a4fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622) -> (continuation of Deferred@1137110874 after retry RPC@709474206) -> (continuation of Deferred@1864980661 after retry RPC@138147721))) after lookup
20:03:15.419 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:15.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:15.420 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:15.421 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.535125 11948 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 12f7da6233f74ebd81a5cab80b3b3648)
20:03:15.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.535519 11948 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:15.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.535666 11949 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 12f7da6233f74ebd81a5cab80b3b3648)
20:03:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.536429 11949 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.537801 11948 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:15.542076 11428 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.545491 11949 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:15.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:15.547036 11428 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:15.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.557653 11321 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b" is_pre_election: true
20:03:15.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:15.558542 11216 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:15.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.558681 11321 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 1.
20:03:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.560715 11425 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b, a32a06b806dc4e41b224f010fdedb7b7; no voters: 12f7da6233f74ebd81a5cab80b3b3648
20:03:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.562436 11948 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:03:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.562858 11948 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 12f7da6233f74ebd81a5cab80b3b3648)
20:03:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.563249 11948 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 1 FOLLOWER]: Advancing to term 2
20:03:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:15.569109 11216 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:15.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.569968 11948 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:15.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.571377 11530 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7" is_pre_election: true
20:03:15.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.572011 11530 raft_consensus.cc:2393] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6719b8a2cff04c88afd580790d9c389b in current term 2: Already voted for candidate a32a06b806dc4e41b224f010fdedb7b7 in this term.
20:03:15.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.572350 11948 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 2 election: Requested vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:15.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.573560 11215 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b; no voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7
20:03:15.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.574513 11949 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 1 FOLLOWER]: Advancing to term 2
20:03:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.574575 11321 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:15.576325 11428 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.579728 11949 raft_consensus.cc:2749] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:03:15.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.584110 11321 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 2.
20:03:15.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.587483 11425 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b, a32a06b806dc4e41b224f010fdedb7b7; no voters: 12f7da6233f74ebd81a5cab80b3b3648
20:03:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.588286 11948 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Leader election won for term 2
20:03:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.589880 11948 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 LEADER]: Becoming Leader. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Running, Role: LEADER
20:03:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.590806 11948 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:15.601199 11017 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 reported cstate change: term changed from 1 to 2, leader changed from 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67) to a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66). New cstate: current_term: 2 leader_uuid: "a32a06b806dc4e41b224f010fdedb7b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:03:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:16.040378 11321 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Refusing update from remote peer a32a06b806dc4e41b224f010fdedb7b7: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:03:16.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:16.041805 11948 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
20:03:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:16.068193 11428 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Couldn't send request to peer 12f7da6233f74ebd81a5cab80b3b3648. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:16.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:16.171303 11604 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9d40 after lost signal to thread 11393
20:03:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:16.900068 11068 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9920 after lost signal to thread 10956
20:03:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:16.900872 11068 debug-util.cc:398] Leaking SignalData structure 0x7b0800097a60 after lost signal to thread 11073
20:03:18.278 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:18.278 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:18.279 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=13, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=4123), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [0ms] delaying RPC due to: Illegal state: connection is terminated, [18ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [21ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [21ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [43ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [46ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [46ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [61ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [63ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [65ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [66ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [98ms] refreshing cache from master, [98ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [102ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [104ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [106ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [106ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [337ms] refreshing cache from master, [338ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [342ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [345ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [347ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [347ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [1257ms] refreshing cache from master, [1258ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1262ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [1263ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [1264ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [1265ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [4117ms] refreshing cache from master, [4118ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [4122ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1413778483(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@e6098f2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622) -> (continuation of Deferred@1137110874 after retry RPC@709474206) -> (continuation of Deferred@1864980661 after retry RPC@138147721) -> (continuation of Deferred@1817975481 after retry RPC@990658308), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a4fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1471942907 after retry RPC@570316381) -> (continuation of Deferred@576165779 after retry RPC@418493622) -> (continuation of Deferred@1137110874 after retry RPC@709474206) -> (continuation of Deferred@1864980661 after retry RPC@138147721) -> (continuation of Deferred@1817975481 after retry RPC@990658308))) after lookup
20:03:18.280 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:18.281 [DEBUG - kudu-nio-2] (Connection.java:757) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] Successfully connected to peer
20:03:18.307 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x0c0b8685, L:/127.0.0.1:51484 - R:/127.10.113.66:34377] using TOKEN/null
20:03:18.361 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.67:37831
20:03:18.363 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:18.365988 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:37831
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32785
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.671582 11428 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Couldn't send request to peer 12f7da6233f74ebd81a5cab80b3b3648. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:18.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.704113 12030 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:18.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.704639 12030 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:18.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.705099 12030 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:18.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.738358 12030 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.739264 12030 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.67
20:03:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:18.777438 12030 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:37831
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32785
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:18.778981 12030 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:18.781247 12030 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.798136 12038 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:20.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:18.799449 12039 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:20.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:20.200940 12037 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 12030
20:03:20.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:20.379021 12030 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.579s	user 0.561s	sys 0.971s
20:03:20.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:20.380877 12030 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.582s	user 0.561s	sys 0.971s
20:03:20.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:20.381379 12042 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:20.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:20.385092 12040 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1580 milliseconds
20:03:20.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.385571 12030 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.386914 12030 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.389554 12030 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:20.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.390993 12030 hybrid_clock.cc:648] HybridClock initialized: now 1786478600390963 us; error 40 us; skew 500 ppm
20:03:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.391888 12030 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:20.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.398094 12030 webserver.cc:533] Webserver started at http://127.10.113.67:32785/ using document root <none> and password file <none>
20:03:20.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.399093 12030 fs_manager.cc:362] Metadata directory not provided
20:03:20.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.399363 12030 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:20.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.407431 12030 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
20:03:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.412484 12057 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:20.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.413787 12030 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
20:03:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.414072 12030 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:11 on dist-test-slave-04bb"
20:03:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.416402 12030 fs_report.cc:389] FS layout report
20:03:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/data
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.467613 12030 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.469211 12030 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.469717 12030 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:20.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.472652 12030 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.478713 12074 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.487115 12030 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.487323 12030 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
20:03:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.487542 12030 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.492136 12030 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.492355 12030 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.001s	sys 0.000s
20:03:20.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.492815 12074 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap starting.
20:03:20.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.555759 12074 log.cc:826] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Log is configured to *not* fsync() on all Append() calls
20:03:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.582423 12074 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.583441 12074 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap complete.
20:03:20.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.585028 12074 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent bootstrapping tablet: real 0.093s	user 0.057s	sys 0.034s
20:03:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.603163 12074 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.605535 12074 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Initialized, Role: FOLLOWER
20:03:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.606822 12074 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.611203 12074 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent starting tablet: real 0.026s	user 0.017s	sys 0.007s
20:03:20.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.681783 12030 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.67:37831
20:03:20.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.682175 12252 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.67:37831 every 8 connection(s)
20:03:20.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.684502 12030 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.694911 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 12030
20:03:20.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:20.736 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:20.737 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.66:34377
20:03:20.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.738196 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 11392
20:03:20.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.752194 12255 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:20.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.752748 12255 heartbeater.cc:461] Registering TS with master...
20:03:20.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.754042 12253 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:20.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.754225 12255 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:20.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.754413 12253 heartbeater.cc:461] Registering TS with master...
20:03:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.755321 12253 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.760751 11018 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.763348 12256 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.763703 12256 heartbeater.cc:461] Registering TS with master...
20:03:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.763837 10887 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.764593 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.67:59839
20:03:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.764612 12256 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:20.768489 10767 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:20.796 [DEBUG - kudu-nio-2] (Connection.java:239) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] handling channelInactive
20:03:20.796 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state READY due to: connection closed
20:03:20.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:20.797 [DEBUG - Time-limited test] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:20.798 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.799 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.799 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.818 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:20.818 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:20.819 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [21ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1173257597(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@203ff4cd -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@681f93b1 -> wakeup thread Time-limited test)) after lookup
20:03:20.820 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:20.821 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.821 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.822 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.837 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:20.837 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:20.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [21ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [23ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [24ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [40ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1173257597(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@203ff4cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@681f93b1 -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764))) after lookup
20:03:20.838 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:20.839 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.839 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.840 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.858 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:20.858 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:20.859 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [21ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [23ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [24ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [40ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [41ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [42ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [43ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [60ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1173257597(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@203ff4cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@681f93b1 -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561))) after lookup
20:03:20.860 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:20.861 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.861 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.861 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.918 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:20.919 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:20.919 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=122), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [21ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [23ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [24ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [40ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [41ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [42ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [43ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [60ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [62ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [64ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [64ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [121ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1173257597(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@203ff4cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@681f93b1 -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583))) after lookup
20:03:20.920 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:20.921 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.921 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:20.922 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:21.078 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:21.079 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:21.079 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=282), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [21ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [23ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [24ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [40ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [41ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [42ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [43ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [60ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [62ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [64ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [64ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [121ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [123ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [125ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [125ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [276ms] refreshing cache from master, [276ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [281ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1173257597(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@203ff4cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583) -> (continuation of Deferred@244927728 after retry RPC@984690450), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@681f93b1 -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583) -> (continuation of Deferred@244927728 after retry RPC@984690450))) after lookup
20:03:21.080 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:21.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:21.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:21.081 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:21.618 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:21.618 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:21.619 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=821), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [21ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [23ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [24ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [40ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [41ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [42ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [43ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [60ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [62ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [64ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [64ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [121ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [123ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [125ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [125ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [276ms] refreshing cache from master, [276ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [281ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [283ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [284ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [284ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [816ms] refreshing cache from master, [816ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [820ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1173257597(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@203ff4cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583) -> (continuation of Deferred@244927728 after retry RPC@984690450) -> (continuation of Deferred@1028790891 after retry RPC@245922668), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@681f93b1 -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583) -> (continuation of Deferred@244927728 after retry RPC@984690450) -> (continuation of Deferred@1028790891 after retry RPC@245922668))) after lookup
20:03:21.619 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:21.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:21.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:21.621 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:21.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.768711 12253 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.848692 12268 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Starting pre-election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:03:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.849177 12268 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:21.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.853699 12268 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:21.855522 11215 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:21.858757 11215 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.862382 12176 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648" is_pre_election: true
20:03:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.863147 12176 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6719b8a2cff04c88afd580790d9c389b in term 1.
20:03:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.864485 11216 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:03:21.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.865223 12268 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Leader pre-election won for term 3
20:03:21.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.865778 12268 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Starting leader election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:03:21.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.866045 12268 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 2 FOLLOWER]: Advancing to term 3
20:03:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.870615 12268 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.872210 12268 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 3 election: Requested vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.873049 12176 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:21.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.873648 12176 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 1 FOLLOWER]: Advancing to term 3
20:03:21.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:21.876181 11215 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.880438 12176 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6719b8a2cff04c88afd580790d9c389b in term 3.
20:03:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.881403 11216 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:03:21.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.881996 12268 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 3 FOLLOWER]: Leader election won for term 3
20:03:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.883118 12268 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 3 LEADER]: Becoming Leader. State: Replica: 6719b8a2cff04c88afd580790d9c389b, State: Running, Role: LEADER
20:03:21.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.884093 12268 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:21.892625 11018 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b reported cstate change: term changed from 2 to 3, leader changed from a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66) to 6719b8a2cff04c88afd580790d9c389b (127.10.113.65). New cstate: current_term: 3 leader_uuid: "6719b8a2cff04c88afd580790d9c389b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: HEALTHY } } }
20:03:22.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:22.470996 12176 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 3 FOLLOWER]: Refusing update from remote peer 6719b8a2cff04c88afd580790d9c389b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
20:03:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:22.472376 12295 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [LEADER]: Connected to new peer: Peer: permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
20:03:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:22.518496 11215 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:24.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:24.870472 11215 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:24.918 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:24.918 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:24.919 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=4122), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [21ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [23ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [24ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [40ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [41ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [42ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [43ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [60ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [62ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [64ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [64ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [117ms] refreshing cache from master, [117ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [121ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [123ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [125ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [125ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [276ms] refreshing cache from master, [276ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [281ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [283ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [284ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [284ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [816ms] refreshing cache from master, [816ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [820ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [822ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [824ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [824ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [4116ms] refreshing cache from master, [4116ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [4121ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1173257597(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@203ff4cd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583) -> (continuation of Deferred@244927728 after retry RPC@984690450) -> (continuation of Deferred@1028790891 after retry RPC@245922668) -> (continuation of Deferred@2120394187 after retry RPC@870109808), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@681f93b1 -> wakeup thread Time-limited test -> (continuation of Deferred@265937813 after retry RPC@458189764) -> (continuation of Deferred@1899156816 after retry RPC@298029561) -> (continuation of Deferred@1812082103 after retry RPC@1631794583) -> (continuation of Deferred@244927728 after retry RPC@984690450) -> (continuation of Deferred@1028790891 after retry RPC@245922668) -> (continuation of Deferred@2120394187 after retry RPC@870109808))) after lookup
20:03:24.919 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:24.920 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] Successfully connected to peer
20:03:24.944 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x654c5d64, L:/127.0.0.1:56708 - R:/127.10.113.65:46157] using TOKEN/null
20:03:25.013 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.66:34377
20:03:25.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:25.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:25.015552 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:25.343223 12350 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:25.343820 12350 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:25.344286 12350 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:25.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:25.379701 12350 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:25.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:25.380856 12350 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.66
20:03:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:25.419312 12350 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:25.420673 12350 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:25.422320 12350 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:25.435596 12363 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:26.886806 11215 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111) [suppressed 10 similar messages]
20:03:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:26.839506 12360 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 12350
20:03:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:27.146335 12360 kernel_stack_watchdog.cc:198] Thread 12350 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:03:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:25.435889 12364 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:27.147715 12350 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.712s	user 0.623s	sys 1.029s
20:03:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:27.148162 12350 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.713s	user 0.623s	sys 1.029s
20:03:27.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:27.149202 12368 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:27.154821 12365 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1711 milliseconds
20:03:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.154918 12350 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.156319 12350 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:27.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.159596 12350 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.160984 12350 hybrid_clock.cc:648] HybridClock initialized: now 1786478607160947 us; error 32 us; skew 500 ppm
20:03:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.162148 12350 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.168253 12350 webserver.cc:533] Webserver started at http://127.10.113.66:33205/ using document root <none> and password file <none>
20:03:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.169322 12350 fs_manager.cc:362] Metadata directory not provided
20:03:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.169579 12350 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:27.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.178076 12350 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
20:03:27.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.182715 12391 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:27.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.183873 12350 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.184185 12350 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:09 on dist-test-slave-04bb"
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.186247 12350 fs_report.cc:389] FS layout report
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/data
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.242058 12350 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.243585 12350 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.244071 12350 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:27.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.246824 12350 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.252971 12404 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.261317 12350 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.261526 12350 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
20:03:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.261747 12350 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:27.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.266414 12350 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:27.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.266592 12350 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
20:03:27.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.267072 12404 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap starting.
20:03:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.336587 12404 log.cc:826] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Log is configured to *not* fsync() on all Append() calls
20:03:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.387127 12404 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.388139 12404 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap complete.
20:03:27.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.389629 12404 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent bootstrapping tablet: real 0.123s	user 0.093s	sys 0.024s
20:03:27.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.406383 12404 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:27.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.409073 12404 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Initialized, Role: FOLLOWER
20:03:27.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.410272 12404 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:27.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.414901 12404 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent starting tablet: real 0.025s	user 0.021s	sys 0.003s
20:03:27.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:27.421648 11215 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:03:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.446708 12350 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.66:34377
20:03:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.447041 12577 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.66:34377 every 8 connection(s)
20:03:27.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.449221 12350 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:27.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.459657 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 12350
20:03:27.461 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:27.491 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:27.493 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.65:46157
20:03:27.493 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:27.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.494277 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 11141
20:03:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.528749 12578 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.529261 12578 heartbeater.cc:461] Registering TS with master...
20:03:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.530534 12578 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.534628 12579 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.534770 11017 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.535018 12579 heartbeater.cc:461] Registering TS with master...
20:03:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.535938 12579 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.538002 11017 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.66:40249
20:03:27.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.540843 10887 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.548799 12582 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.549153 12582 heartbeater.cc:461] Registering TS with master...
20:03:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.549772 12582 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:27.552 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] handling channelInactive
20:03:27.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state READY due to: connection closed
20:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:27.555189 10767 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:27.557 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:27.558 [DEBUG - Time-limited test] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:27.559 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.560 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.560 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.577 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:27.578 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:27.578 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [19ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@420623193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3a86818b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c9a0e79 -> wakeup thread Time-limited test)) after lookup
20:03:27.579 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:27.580 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.580 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.580 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.597 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:27.598 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:27.598 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [19ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [21ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [22ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [39ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@420623193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3a86818b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c9a0e79 -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536))) after lookup
20:03:27.599 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:27.600 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.600 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.600 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.638 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:27.638 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:27.638 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [19ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [21ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [22ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [39ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [40ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [42ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [79ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@420623193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3a86818b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c9a0e79 -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708))) after lookup
20:03:27.639 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:27.640 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.640 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.640 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.778 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:27.778 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:27.779 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=221), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [19ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [21ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [22ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [39ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [40ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [42ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [79ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [81ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [82ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [82ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [220ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@420623193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3a86818b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708) -> (continuation of Deferred@1941328033 after retry RPC@2062057522), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c9a0e79 -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708) -> (continuation of Deferred@1941328033 after retry RPC@2062057522))) after lookup
20:03:27.779 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:27.780 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.781 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:27.781 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:28.078 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:28.078 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:28.078 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=520), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [19ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [21ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [22ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [39ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [40ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [42ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [79ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [81ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [82ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [82ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [220ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [221ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [223ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [223ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [515ms] refreshing cache from master, [515ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [519ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@420623193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3a86818b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708) -> (continuation of Deferred@1941328033 after retry RPC@2062057522) -> (continuation of Deferred@1941002331 after retry RPC@1298579215), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c9a0e79 -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708) -> (continuation of Deferred@1941328033 after retry RPC@2062057522) -> (continuation of Deferred@1941002331 after retry RPC@1298579215))) after lookup
20:03:28.079 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:28.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:28.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:28.080 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.542294 12578 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:28.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.778290 12601 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.778855 12601 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.781507 12601 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:28.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:28.785586 12392 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:28.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:28.790445 12392 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.795176 12176 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648" is_pre_election: true
20:03:28.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.797143 12395 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a32a06b806dc4e41b224f010fdedb7b7; no voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b
20:03:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.797533 12607 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 6719b8a2cff04c88afd580790d9c389b)
20:03:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.797799 12601 raft_consensus.cc:2749] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
20:03:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.798142 12607 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.801072 12607 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:28.804785 12058 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:28.809723 12058 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.814031 12501 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7" is_pre_election: true
20:03:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.814663 12501 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 2.
20:03:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.815971 12059 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7; no voters: 6719b8a2cff04c88afd580790d9c389b
20:03:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.816834 12607 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 3 FOLLOWER]: Leader pre-election won for term 4
20:03:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.817220 12607 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 6719b8a2cff04c88afd580790d9c389b)
20:03:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.817574 12607 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 3 FOLLOWER]: Advancing to term 4
20:03:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.822489 12607 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.824260 12607 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 4 election: Requested vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.825064 12501 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.825564 12501 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 2 FOLLOWER]: Advancing to term 4
20:03:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:28.828083 12058 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.832469 12501 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 4.
20:03:28.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.833537 12059 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7; no voters: 6719b8a2cff04c88afd580790d9c389b
20:03:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.834224 12607 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 4 FOLLOWER]: Leader election won for term 4
20:03:28.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.835839 12607 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 4 LEADER]: Becoming Leader. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Running, Role: LEADER
20:03:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.837276 12607 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:28.847678 11017 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 reported cstate change: term changed from 3 to 4, leader changed from 6719b8a2cff04c88afd580790d9c389b (127.10.113.65) to 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67). New cstate: current_term: 4 leader_uuid: "12f7da6233f74ebd81a5cab80b3b3648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:03:28.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:28.858404 10809 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1de0 after lost signal to thread 10711
20:03:28.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:28.859412 10809 debug-util.cc:398] Leaking SignalData structure 0x7b08000abaa0 after lost signal to thread 10812
20:03:29.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:29.259120 12501 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 4 FOLLOWER]: Refusing update from remote peer 12f7da6233f74ebd81a5cab80b3b3648: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
20:03:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:29.260594 12607 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
20:03:29.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:29.308122 12058 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 -> Peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Couldn't send request to peer 6719b8a2cff04c88afd580790d9c389b. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:29.438 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:29.438 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:29.439 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1881), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [19ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [21ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [22ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [22ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [39ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [40ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [42ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [42ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] refreshing cache from master, [75ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [79ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [81ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [82ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [82ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [216ms] refreshing cache from master, [216ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [220ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [221ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [223ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [223ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [515ms] refreshing cache from master, [515ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [519ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [521ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [522ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [522ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [1876ms] refreshing cache from master, [1876ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1880ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@420623193(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3a86818b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708) -> (continuation of Deferred@1941328033 after retry RPC@2062057522) -> (continuation of Deferred@1941002331 after retry RPC@1298579215) -> (continuation of Deferred@943201908 after retry RPC@1079922875), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c9a0e79 -> wakeup thread Time-limited test -> (continuation of Deferred@1818282179 after retry RPC@1354103536) -> (continuation of Deferred@822470486 after retry RPC@1310259708) -> (continuation of Deferred@1941328033 after retry RPC@2062057522) -> (continuation of Deferred@1941002331 after retry RPC@1298579215) -> (continuation of Deferred@943201908 after retry RPC@1079922875))) after lookup
20:03:29.440 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:29.441 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] Successfully connected to peer
20:03:29.474 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x520d5a1a, L:/127.0.0.1:39416 - R:/127.10.113.67:37831] using TOKEN/null
20:03:29.532 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.65:46157
20:03:29.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:29.534377 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:46157
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42403
20:03:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:29.929875 12653 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:29.930359 12653 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:29.930975 12653 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:29.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:29.966360 12653 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:29.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:29.967371 12653 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.65
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:30.005615 12653 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:46157
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42403
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:30.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:30.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:30.007382 12653 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:30.009524 12653 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:30.023283 12666 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:31.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:31.838893 12058 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 -> Peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Couldn't send request to peer 6719b8a2cff04c88afd580790d9c389b. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:32.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:31.428498 12663 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 12653
20:03:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:30.025108 12667 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:32.023623 12653 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.999s	user 0.642s	sys 1.217s
20:03:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:32.024185 12653 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.000s	user 0.642s	sys 1.217s
20:03:32.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:32.027192 12669 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:32.029430 12668 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2000 milliseconds
20:03:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.029459 12653 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:32.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.030898 12653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:32.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.033298 12653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:32.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.034677 12653 hybrid_clock.cc:648] HybridClock initialized: now 1786478612034625 us; error 53 us; skew 500 ppm
20:03:32.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.035552 12653 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.041723 12653 webserver.cc:533] Webserver started at http://127.10.113.65:42403/ using document root <none> and password file <none>
20:03:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.042923 12653 fs_manager.cc:362] Metadata directory not provided
20:03:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.043226 12653 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:32.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.051301 12653 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
20:03:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.056207 12690 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.057382 12653 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
20:03:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.057767 12653 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:06 on dist-test-slave-04bb"
20:03:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.059733 12653 fs_report.cc:389] FS layout report
20:03:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/data
20:03:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.114812 12653 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.116387 12653 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.117003 12653 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.119581 12653 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.125590 12707 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:32.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.133914 12653 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:32.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.134166 12653 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
20:03:32.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.134552 12653 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:32.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.139160 12653 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:32.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.139372 12653 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
20:03:32.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.139694 12707 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap starting.
20:03:32.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.215852 12707 log.cc:826] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Log is configured to *not* fsync() on all Append() calls
20:03:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.262069 12707 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.263656 12707 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap complete.
20:03:32.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.265450 12707 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent bootstrapping tablet: real 0.126s	user 0.086s	sys 0.038s
20:03:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.287133 12707 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.290349 12707 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6719b8a2cff04c88afd580790d9c389b, State: Initialized, Role: FOLLOWER
20:03:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.291807 12707 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:32.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.296567 12707 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent starting tablet: real 0.031s	user 0.024s	sys 0.004s
20:03:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.333889 12653 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.65:46157
20:03:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.334273 12885 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.65:46157 every 8 connection(s)
20:03:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.337548 12653 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:32.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.347399 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 12653
20:03:32.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:32.397 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:32.404 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.67:37831
20:03:32.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.399299 12886 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:32.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.399821 12886 heartbeater.cc:461] Registering TS with master...
20:03:32.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:32.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.401198 12886 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.405045 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 12030
20:03:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.405473 11018 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.408612 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.65:56881
20:03:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.415118 12888 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.415518 12888 heartbeater.cc:461] Registering TS with master...
20:03:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.416621 12888 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:32.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.420096 10887 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:32.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.424505 12894 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:32.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.424890 12894 heartbeater.cc:461] Registering TS with master...
20:03:32.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.425520 12894 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:32.428742 10765 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:32.461 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] handling channelInactive
20:03:32.462 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state READY due to: connection closed
20:03:32.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:32.464 [DEBUG - Time-limited test] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:32.466 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.466 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.466 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.478 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:32.478 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:32.479 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@411379475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@eb1af6b -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ba8ec6d -> wakeup thread Time-limited test)) after lookup
20:03:32.479 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:32.480 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.481 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.497 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:32.497 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:32.497 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@411379475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@eb1af6b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ba8ec6d -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532))) after lookup
20:03:32.498 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:32.499 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.499 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.500 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.537 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:32.538 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:32.538 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [34ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [35ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [73ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@411379475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@eb1af6b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ba8ec6d -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056))) after lookup
20:03:32.538 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:32.539 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.539 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.540 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.677 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:32.678 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:32.678 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=214), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [34ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [35ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [73ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [74ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [75ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [76ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [213ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@411379475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@eb1af6b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056) -> (continuation of Deferred@2131981432 after retry RPC@1917186475), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ba8ec6d -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056) -> (continuation of Deferred@2131981432 after retry RPC@1917186475))) after lookup
20:03:32.679 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:32.680 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.680 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.680 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.716 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:32.717 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:32.717 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=253), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [34ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [35ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [73ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [74ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [75ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [76ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [213ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [214ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [216ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [216ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [249ms] refreshing cache from master, [249ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [252ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@411379475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@eb1af6b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056) -> (continuation of Deferred@2131981432 after retry RPC@1917186475) -> (continuation of Deferred@324731707 after retry RPC@1313759314), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ba8ec6d -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056) -> (continuation of Deferred@2131981432 after retry RPC@1917186475) -> (continuation of Deferred@324731707 after retry RPC@1313759314))) after lookup
20:03:32.718 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:32.719 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.719 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:32.719 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:33.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.417774 12886 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.567341 12908 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 12f7da6233f74ebd81a5cab80b3b3648)
20:03:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.567754 12908 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.569519 12908 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:33.575874 12395 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.583124 12811 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b" is_pre_election: true
20:03:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.583890 12811 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 3.
20:03:33.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.585562 12392 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b, a32a06b806dc4e41b224f010fdedb7b7; no voters: 12f7da6233f74ebd81a5cab80b3b3648
20:03:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.586287 12908 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 4 FOLLOWER]: Leader pre-election won for term 5
20:03:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.586535 12908 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 12f7da6233f74ebd81a5cab80b3b3648)
20:03:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.586781 12908 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 4 FOLLOWER]: Advancing to term 5
20:03:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.591632 12908 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.593092 12908 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 5 election: Requested vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:33.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.594234 12811 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.594740 12811 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 3 FOLLOWER]: Advancing to term 5
20:03:33.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:33.596587 12395 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:33.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.601400 12811 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 5.
20:03:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.602211 12392 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b, a32a06b806dc4e41b224f010fdedb7b7; no voters: 12f7da6233f74ebd81a5cab80b3b3648
20:03:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.602792 12908 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 5 FOLLOWER]: Leader election won for term 5
20:03:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.603888 12908 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 5 LEADER]: Becoming Leader. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Running, Role: LEADER
20:03:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.604802 12908 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:33.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:33.614403 11018 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 reported cstate change: term changed from 4 to 5, leader changed from 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67) to a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66). New cstate: current_term: 5 leader_uuid: "a32a06b806dc4e41b224f010fdedb7b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:03:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.130276 12395 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111) [suppressed 2 similar messages]
20:03:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.133481 12395 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Couldn't send request to peer 12f7da6233f74ebd81a5cab80b3b3648. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:34.190001 12811 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 5 FOLLOWER]: Refusing update from remote peer a32a06b806dc4e41b224f010fdedb7b7: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
20:03:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:34.191624 12939 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
20:03:34.238 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:34.239 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:34.239 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1775), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [34ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [35ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [70ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [73ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [74ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [75ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [76ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [213ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [214ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [216ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [216ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [249ms] refreshing cache from master, [249ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [252ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [253ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [255ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [255ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [1769ms] refreshing cache from master, [1770ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1774ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@411379475(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@eb1af6b -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056) -> (continuation of Deferred@2131981432 after retry RPC@1917186475) -> (continuation of Deferred@324731707 after retry RPC@1313759314) -> (continuation of Deferred@2015847342 after retry RPC@1171705864), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7ba8ec6d -> wakeup thread Time-limited test -> (continuation of Deferred@1146590687 after retry RPC@2023801532) -> (continuation of Deferred@1361042509 after retry RPC@794051056) -> (continuation of Deferred@2131981432 after retry RPC@1917186475) -> (continuation of Deferred@324731707 after retry RPC@1313759314) -> (continuation of Deferred@2015847342 after retry RPC@1171705864))) after lookup
20:03:34.240 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:34.241 [DEBUG - kudu-nio-0] (Connection.java:757) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] Successfully connected to peer
20:03:34.258 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x1d8ed8a7, L:/127.0.0.1:42116 - R:/127.10.113.66:34377] using TOKEN/null
20:03:34.301 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.67:37831
20:03:34.301 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:34.302510 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:37831
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32785
20:03:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.645843 12965 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.646327 12965 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.646766 12965 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.681314 12965 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:34.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.682449 12965 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.67
20:03:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:34.722205 12965 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:37831
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32785
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:34.723965 12965 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:34.726320 12965 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.742355 12982 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:36.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:34.743458 12984 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:36.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:36.147178 12980 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 12965
20:03:36.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:36.575173 12965 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.832s	user 0.675s	sys 1.071s
20:03:36.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:36.575557 12965 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.832s	user 0.675s	sys 1.071s
20:03:36.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:36.577826 12988 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:36.584949 12985 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1841 milliseconds
20:03:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.584959 12965 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.586740 12965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.589109 12965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:36.589747 12395 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Couldn't send request to peer 12f7da6233f74ebd81a5cab80b3b3648. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.590552 12965 hybrid_clock.cc:648] HybridClock initialized: now 1786478616590508 us; error 39 us; skew 500 ppm
20:03:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.591436 12965 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.598174 12965 webserver.cc:533] Webserver started at http://127.10.113.67:32785/ using document root <none> and password file <none>
20:03:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.599180 12965 fs_manager.cc:362] Metadata directory not provided
20:03:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.599426 12965 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:36.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.607807 12965 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
20:03:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.613121 13013 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.614426 12965 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
20:03:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.614792 12965 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:11 on dist-test-slave-04bb"
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.617436 12965 fs_report.cc:389] FS layout report
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/data
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:36.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.680966 12965 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.682510 12965 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:36.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.682986 12965 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:36.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.685612 12965 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:36.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.691424 13026 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.700171 12965 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.700419 12965 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
20:03:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.700688 12965 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:36.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.705221 12965 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:36.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.705402 12965 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
20:03:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.705965 13026 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap starting.
20:03:36.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.770543 13026 log.cc:826] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Log is configured to *not* fsync() on all Append() calls
20:03:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.850674 13026 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.852010 13026 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap complete.
20:03:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.854487 13026 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent bootstrapping tablet: real 0.149s	user 0.094s	sys 0.050s
20:03:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.885615 13026 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.889842 13026 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Initialized, Role: FOLLOWER
20:03:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.891235 13026 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.895771 13026 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent starting tablet: real 0.041s	user 0.028s	sys 0.012s
20:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.899447 12965 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.67:37831
20:03:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.899709 13211 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.67:37831 every 8 connection(s)
20:03:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.902184 12965 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.908313 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 12965
20:03:36.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:36.954 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:36.955 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.66:34377
20:03:36.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:36.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.956247 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 12350
20:03:36.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.981911 13213 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:36.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.982491 13214 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.982915 13214 heartbeater.cc:461] Registering TS with master...
20:03:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.982455 13213 heartbeater.cc:461] Registering TS with master...
20:03:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.983507 13212 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.983891 13212 heartbeater.cc:461] Registering TS with master...
20:03:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.984622 13213 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.984905 13212 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.985275 13214 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.989986 10767 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.990022 10887 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.991595 11017 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:36.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:36.994117 11017 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.67:59073
20:03:37.013 [DEBUG - kudu-nio-0] (Connection.java:239) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] handling channelInactive
20:03:37.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state READY due to: connection closed
20:03:37.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:37.015 [DEBUG - Time-limited test] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:37.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.016 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.037 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:37.038 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:37.038 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [23ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1315048780(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3d47d218 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a3d1f0 -> wakeup thread Time-limited test)) after lookup
20:03:37.038 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:37.039 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.039 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.039 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.057 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:37.057 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:37.057 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [23ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [24ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [25ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1315048780(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3d47d218 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a3d1f0 -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706))) after lookup
20:03:37.058 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:37.059 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.059 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.059 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.098 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:37.098 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:37.099 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [23ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [24ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [25ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [44ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [45ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [45ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [84ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1315048780(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3d47d218 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a3d1f0 -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309))) after lookup
20:03:37.100 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:37.101 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.101 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.101 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.178 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:37.179 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:37.179 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=164), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [23ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [24ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [25ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [44ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [45ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [45ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [84ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [85ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [87ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [87ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [160ms] refreshing cache from master, [160ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [164ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1315048780(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3d47d218 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a3d1f0 -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714))) after lookup
20:03:37.180 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:37.181 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.181 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.181 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.678 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:37.678 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:37.679 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=664), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [23ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [24ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [25ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [44ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [45ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [45ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [84ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [85ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [87ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [87ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [160ms] refreshing cache from master, [160ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [164ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [166ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [167ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [168ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [660ms] refreshing cache from master, [660ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [664ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1315048780(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3d47d218 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714) -> (continuation of Deferred@1954223415 after retry RPC@82662467), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a3d1f0 -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714) -> (continuation of Deferred@1954223415 after retry RPC@82662467))) after lookup
20:03:37.680 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:37.681 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.682 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.682 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.737 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:37.738 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:37.738 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=723), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [23ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [24ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [25ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [44ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [45ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [45ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [84ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [85ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [87ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [87ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [160ms] refreshing cache from master, [160ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [164ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [166ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [167ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [168ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [660ms] refreshing cache from master, [660ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [664ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [666ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [668ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [668ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [719ms] refreshing cache from master, [720ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [723ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1315048780(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3d47d218 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714) -> (continuation of Deferred@1954223415 after retry RPC@82662467) -> (continuation of Deferred@567223676 after retry RPC@1341006410), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a3d1f0 -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714) -> (continuation of Deferred@1954223415 after retry RPC@82662467) -> (continuation of Deferred@567223676 after retry RPC@1341006410))) after lookup
20:03:37.738 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:37.739 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.740 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.740 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:37.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:37.997341 13212 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:38.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.082430 13238 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 5 FOLLOWER]: Starting pre-election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:03:38.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.082957 13238 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.085632 13238 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:38.089398 12697 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:38.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:38.093650 12697 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:38.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.098613 13133 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648" is_pre_election: true
20:03:38.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.099442 13133 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6719b8a2cff04c88afd580790d9c389b in term 4.
20:03:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.101233 12698 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:03:38.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.101954 13238 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 5 FOLLOWER]: Leader pre-election won for term 6
20:03:38.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.102295 13238 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 5 FOLLOWER]: Starting leader election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:03:38.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.102658 13238 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 5 FOLLOWER]: Advancing to term 6
20:03:38.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.107937 13238 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.109846 13238 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 6 election: Requested vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.110692 13133 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.111312 13133 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 4 FOLLOWER]: Advancing to term 6
20:03:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:38.113767 12697 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.118628 13133 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6719b8a2cff04c88afd580790d9c389b in term 6.
20:03:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.119529 12698 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:03:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.120237 13238 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 6 FOLLOWER]: Leader election won for term 6
20:03:38.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.121865 13238 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 6 LEADER]: Becoming Leader. State: Replica: 6719b8a2cff04c88afd580790d9c389b, State: Running, Role: LEADER
20:03:38.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.122704 13238 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:38.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.135149 11017 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b reported cstate change: term changed from 5 to 6, leader changed from a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66) to 6719b8a2cff04c88afd580790d9c389b (127.10.113.65). New cstate: current_term: 6 leader_uuid: "6719b8a2cff04c88afd580790d9c389b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: HEALTHY } } }
20:03:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:38.541610 12697 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.594937 13133 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 6 FOLLOWER]: Refusing update from remote peer 6719b8a2cff04c88afd580790d9c389b: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
20:03:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:38.596402 13238 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [LEADER]: Connected to new peer: Peer: permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
20:03:40.238 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:40.238 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:40.238 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=3223), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [23ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [24ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [25ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [25ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [42ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [44ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [45ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [45ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [84ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [85ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [87ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [87ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [160ms] refreshing cache from master, [160ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [164ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [166ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [167ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [168ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [660ms] refreshing cache from master, [660ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [664ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [666ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [668ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [668ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [719ms] refreshing cache from master, [720ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [723ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [724ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [726ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [726ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [3219ms] refreshing cache from master, [3220ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [3223ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1315048780(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@3d47d218 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714) -> (continuation of Deferred@1954223415 after retry RPC@82662467) -> (continuation of Deferred@567223676 after retry RPC@1341006410) -> (continuation of Deferred@1029866625 after retry RPC@60820757), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a3d1f0 -> wakeup thread Time-limited test -> (continuation of Deferred@126373771 after retry RPC@2050027706) -> (continuation of Deferred@413819505 after retry RPC@196489309) -> (continuation of Deferred@195478579 after retry RPC@824715714) -> (continuation of Deferred@1954223415 after retry RPC@82662467) -> (continuation of Deferred@567223676 after retry RPC@1341006410) -> (continuation of Deferred@1029866625 after retry RPC@60820757))) after lookup
20:03:40.239 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:40.240 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] Successfully connected to peer
20:03:40.261 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xa861b784, L:/127.0.0.1:57856 - R:/127.10.113.65:46157] using TOKEN/null
20:03:40.317 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.66:34377
20:03:40.319 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:40.320874 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.675800 13304 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.676391 13304 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:40.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.677083 13304 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.710300 13304 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.711295 13304 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.66
20:03:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:40.748127 13304 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:40.749915 13304 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:40.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:40.751868 13304 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.765596 13312 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.794167 12697 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:42.168653 13311 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 13304
20:03:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:42.482200 13304 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.716s	user 0.582s	sys 1.108s
20:03:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:42.482618 13304 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.716s	user 0.582s	sys 1.108s
20:03:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:40.766860 13318 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:42.485139 13321 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:42.489970 13319 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1722 milliseconds
20:03:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.490027 13304 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:42.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.491369 13304 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.493664 13304 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.495056 13304 hybrid_clock.cc:648] HybridClock initialized: now 1786478622495016 us; error 38 us; skew 500 ppm
20:03:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.495947 13304 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.502225 13304 webserver.cc:533] Webserver started at http://127.10.113.66:33205/ using document root <none> and password file <none>
20:03:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.503245 13304 fs_manager.cc:362] Metadata directory not provided
20:03:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.503530 13304 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:42.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.512239 13304 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.000s
20:03:42.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.517306 13336 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:42.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.518481 13304 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.518883 13304 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:09 on dist-test-slave-04bb"
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.521032 13304 fs_report.cc:389] FS layout report
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/data
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.584331 13304 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.586005 13304 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.586575 13304 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.589407 13304 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.595152 13345 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.603662 13304 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.603938 13304 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
20:03:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.604332 13304 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.609429 13304 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.609644 13304 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
20:03:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.610083 13345 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap starting.
20:03:42.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.694375 13345 log.cc:826] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Log is configured to *not* fsync() on all Append() calls
20:03:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.764045 13345 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:42.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.765326 13345 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap complete.
20:03:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.767344 13345 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent bootstrapping tablet: real 0.158s	user 0.110s	sys 0.042s
20:03:42.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.791097 13345 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:42.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.794173 13345 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Initialized, Role: FOLLOWER
20:03:42.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.795182 13345 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.799430 13345 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent starting tablet: real 0.032s	user 0.029s	sys 0.003s
20:03:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.826967 13304 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.66:34377
20:03:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.827324 13545 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.66:34377 every 8 connection(s)
20:03:42.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.829949 13304 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.833411 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 13304
20:03:42.834 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:42.910 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:42.912 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.65:46157
20:03:42.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:42.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.909765 13547 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:42.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.910311 13547 heartbeater.cc:461] Registering TS with master...
20:03:42.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.911636 13547 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:42.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.913606 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 12653
20:03:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.929381 13546 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.929781 13546 heartbeater.cc:461] Registering TS with master...
20:03:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.930655 13546 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.931245 10887 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:42.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.934173 11018 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:42.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.937304 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.66:59989
20:03:42.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.943473 13549 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:42.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.943957 13549 heartbeater.cc:461] Registering TS with master...
20:03:42.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.944725 13549 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:42.948572 10765 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:42.967 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] handling channelInactive
20:03:42.967 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state READY due to: connection closed
20:03:42.967 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:42.968 [DEBUG - Time-limited test] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:42.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.970 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:42.970763 13558 negotiation.cc:336] Failed RPC negotiation. Trace:
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.852788 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.10.113.65:38639 (local address 127.10.113.66:34377)
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.891098 (+ 38310us) server_negotiation.cc:207] Beginning negotiation
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.891137 (+    39us) server_negotiation.cc:400] Waiting for connection header
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.891221 (+    84us) server_negotiation.cc:408] Connection header received
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.891504 (+   283us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.891550 (+    46us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.892066 (+   516us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.892397 (+   331us) server_negotiation.cc:227] Negotiated authn=SASL
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.898682 (+  6285us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.909122 (+ 10440us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.914122 (+  5000us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.914684 (+   562us) server_negotiation.cc:658] Negotiated auth-only TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:03:42.968264 (+ 53580us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.10.113.65:38639: BlockingRecv error: recv got EOF from 127.10.113.65:38639 (error 108)
20:03:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":37362,"thread_start_us":39603,"threads_started":1}
20:03:42.977 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:42.977 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:42.977 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [1ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [9ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1210261477(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@4be86b51 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d862ec2 -> wakeup thread Time-limited test)) after lookup
20:03:42.978 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:42.979 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.979 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.979 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.997 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:42.997 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:42.998 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [1ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [9ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [10ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [11ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [29ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1210261477(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@4be86b51 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d862ec2 -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359))) after lookup
20:03:42.998 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:42.999 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.999 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:42.999 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.017 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:43.017 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:43.018 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [1ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [9ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [10ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [11ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [29ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [30ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [31ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [31ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [49ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1210261477(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@4be86b51 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d862ec2 -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953))) after lookup
20:03:43.018 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:43.019 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.020 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.020 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.077 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:43.078 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:43.078 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=110), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [1ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [9ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [10ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [11ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [29ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [30ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [31ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [31ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [49ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [50ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [52ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [52ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [109ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1210261477(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@4be86b51 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953) -> (continuation of Deferred@1677370794 after retry RPC@447032029), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d862ec2 -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953) -> (continuation of Deferred@1677370794 after retry RPC@447032029))) after lookup
20:03:43.078 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:43.079 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.079 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.080 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.258 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:43.258 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:43.258 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=290), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [1ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [9ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [10ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [11ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [29ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [30ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [31ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [31ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [49ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [50ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [52ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [52ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [109ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [110ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [111ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [112ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [286ms] refreshing cache from master, [286ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [289ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1210261477(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@4be86b51 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953) -> (continuation of Deferred@1677370794 after retry RPC@447032029) -> (continuation of Deferred@1160813062 after retry RPC@1995482047), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d862ec2 -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953) -> (continuation of Deferred@1677370794 after retry RPC@447032029) -> (continuation of Deferred@1160813062 after retry RPC@1995482047))) after lookup
20:03:43.259 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:43.260 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.260 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.260 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:43.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.941433 13546 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:43.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.943104 13566 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 6719b8a2cff04c88afd580790d9c389b)
20:03:43.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.943979 13566 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:43.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.946525 13566 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:43.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:43.950515 13016 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:43.955614 13016 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:43.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.958706 13466 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7" is_pre_election: true
20:03:43.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.959386 13466 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 5.
20:03:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.960700 13017 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7; no voters: 6719b8a2cff04c88afd580790d9c389b
20:03:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.961380 13566 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 6 FOLLOWER]: Leader pre-election won for term 7
20:03:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.961704 13566 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 6719b8a2cff04c88afd580790d9c389b)
20:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.961951 13566 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 6 FOLLOWER]: Advancing to term 7
20:03:43.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.966416 13566 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.967914 13566 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 7 election: Requested vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.968683 13466 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.969239 13466 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 5 FOLLOWER]: Advancing to term 7
20:03:43.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:43.971774 13016 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:43.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.975890 13466 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 7.
20:03:43.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.976810 13017 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7; no voters: 6719b8a2cff04c88afd580790d9c389b
20:03:43.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.977370 13566 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 7 FOLLOWER]: Leader election won for term 7
20:03:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.978863 13566 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 7 LEADER]: Becoming Leader. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Running, Role: LEADER
20:03:43.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.980355 13566 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:43.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:43.988981 11018 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 reported cstate change: term changed from 6 to 7, leader changed from 6719b8a2cff04c88afd580790d9c389b (127.10.113.65) to 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67). New cstate: current_term: 7 leader_uuid: "12f7da6233f74ebd81a5cab80b3b3648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:03:44.178 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:44.178 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:44.178 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1210), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [1ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [9ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [10ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [11ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [11ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [29ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [30ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [31ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [31ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [49ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [50ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [52ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [52ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [109ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [110ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [111ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [112ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [286ms] refreshing cache from master, [286ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [289ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [291ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [292ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [292ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [1205ms] refreshing cache from master, [1205ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1209ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@1210261477(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@4be86b51 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953) -> (continuation of Deferred@1677370794 after retry RPC@447032029) -> (continuation of Deferred@1160813062 after retry RPC@1995482047) -> (continuation of Deferred@51352954 after retry RPC@1880175615), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4d862ec2 -> wakeup thread Time-limited test -> (continuation of Deferred@1003241848 after retry RPC@1503066359) -> (continuation of Deferred@444318913 after retry RPC@1959056953) -> (continuation of Deferred@1677370794 after retry RPC@447032029) -> (continuation of Deferred@1160813062 after retry RPC@1995482047) -> (continuation of Deferred@51352954 after retry RPC@1880175615))) after lookup
20:03:44.179 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:44.180 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] Successfully connected to peer
20:03:44.213 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0xbf694e9f, L:/127.0.0.1:44146 - R:/127.10.113.67:37831] using TOKEN/null
20:03:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:44.224910 13466 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 7 FOLLOWER]: Refusing update from remote peer 12f7da6233f74ebd81a5cab80b3b3648: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
20:03:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:44.226889 13572 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.001s
20:03:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.229691 13016 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 -> Peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Couldn't send request to peer 6719b8a2cff04c88afd580790d9c389b. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:44.293 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.65:46157
20:03:44.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:44.295104 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:46157
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42403
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:44.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.623700 13607 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:44.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.624581 13607 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:44.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.625836 13607 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.690734 13607 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.691560 13607 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.65
20:03:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:44.729473 13607 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:46157
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42403
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:44.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:44.730798 13607 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:44.732877 13607 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.746208 13620 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:44.747313 13621 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:46.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:46.151890 13624 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:46.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:46.156028 13622 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1408 milliseconds
20:03:46.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:46.157557 13607 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.411s	user 0.407s	sys 0.965s
20:03:46.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:46.157835 13607 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.411s	user 0.407s	sys 0.965s
20:03:46.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.158020 13607 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:46.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.159087 13607 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.161419 13607 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:46.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.162791 13607 hybrid_clock.cc:648] HybridClock initialized: now 1786478626162758 us; error 45 us; skew 500 ppm
20:03:46.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.163692 13607 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:46.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.171510 13607 webserver.cc:533] Webserver started at http://127.10.113.65:42403/ using document root <none> and password file <none>
20:03:46.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.172724 13607 fs_manager.cc:362] Metadata directory not provided
20:03:46.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.172995 13607 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.182265 13607 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.004s	sys 0.004s
20:03:46.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.195150 13641 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.196852 13607 fs_manager.cc:730] Time spent opening block manager: real 0.012s	user 0.005s	sys 0.006s
20:03:46.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.197280 13607 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:46.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:46.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:06 on dist-test-slave-04bb"
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.199441 13607 fs_report.cc:389] FS layout report
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/data
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:46.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.307605 13607 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:46.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.309711 13607 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:46.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.310134 13607 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:46.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.313043 13607 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:46.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.318910 13656 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.328655 13607 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.328878 13607 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.001s	sys 0.001s
20:03:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.329093 13607 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.333918 13607 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.334153 13607 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
20:03:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.334532 13656 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap starting.
20:03:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.398782 13656 log.cc:826] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Log is configured to *not* fsync() on all Append() calls
20:03:46.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.497241 13656 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:46.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.498441 13656 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap complete.
20:03:46.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.500561 13656 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent bootstrapping tablet: real 0.166s	user 0.113s	sys 0.047s
20:03:46.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.516449 13607 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.65:46157
20:03:46.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.516628 13828 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.65:46157 every 8 connection(s)
20:03:46.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.520026 13607 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.524001 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 13607
20:03:46.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.529001 13656 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.532258 13656 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6719b8a2cff04c88afd580790d9c389b, State: Initialized, Role: FOLLOWER
20:03:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.533282 13656 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:46.573 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:46.574 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.67:37831
20:03:46.574 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:46.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.575009 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 12965
20:03:46.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.594444 13656 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent starting tablet: real 0.094s	user 0.042s	sys 0.025s
20:03:46.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.601931 13833 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:46.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.602380 13833 heartbeater.cc:461] Registering TS with master...
20:03:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.603744 13830 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.603758 13833 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:46.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.604120 13830 heartbeater.cc:461] Registering TS with master...
20:03:46.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.605113 13830 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:46.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.608299 11018 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:46.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.608785 10887 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:46.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.611505 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.65:57055
20:03:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.615222 13834 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.615574 13834 heartbeater.cc:461] Registering TS with master...
20:03:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.616288 13834 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.619278 13830 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:46.619971 10765 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:46.641 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] handling channelInactive
20:03:46.641 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state READY due to: connection closed
20:03:46.643 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:46.643 [DEBUG - Time-limited test] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:46.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.645 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.657 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:46.658 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:46.658 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@51644892(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@7a634c8a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b964c06 -> wakeup thread Time-limited test)) after lookup
20:03:46.659 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:46.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.660 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.677 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:46.677 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:46.677 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [34ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@51644892(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@7a634c8a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b964c06 -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176))) after lookup
20:03:46.678 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:46.679 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.679 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.679 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.696 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:46.697 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:46.697 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [34ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [35ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [36ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [53ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@51644892(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@7a634c8a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b964c06 -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152))) after lookup
20:03:46.698 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:46.699 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.699 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.699 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.717 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:46.718 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:46.718 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [34ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [35ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [36ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [53ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [54ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [56ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [56ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [74ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@51644892(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@7a634c8a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152) -> (continuation of Deferred@389386297 after retry RPC@1454117993), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b964c06 -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152) -> (continuation of Deferred@389386297 after retry RPC@1454117993))) after lookup
20:03:46.718 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:46.719 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.719 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.719 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.758 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:46.758 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:46.758 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=115), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [34ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [35ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [36ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [53ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [54ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [56ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [56ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [74ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [75ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [76ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [76ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [114ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@51644892(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@7a634c8a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152) -> (continuation of Deferred@389386297 after retry RPC@1454117993) -> (continuation of Deferred@1862077502 after retry RPC@1489261268), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b964c06 -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152) -> (continuation of Deferred@389386297 after retry RPC@1454117993) -> (continuation of Deferred@1862077502 after retry RPC@1489261268))) after lookup
20:03:46.759 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:03:46.760 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.761 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state CONNECTING due to: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:46.761 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831
20:03:48.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.052434 13849 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 12f7da6233f74ebd81a5cab80b3b3648)
20:03:48.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.052980 13849 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:48.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.055508 13849 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:48.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.058478 13340 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:48.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.063584 13340 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.067560 13753 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b" is_pre_election: true
20:03:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.068234 13753 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 6.
20:03:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.069681 13338 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b, a32a06b806dc4e41b224f010fdedb7b7; no voters: 12f7da6233f74ebd81a5cab80b3b3648
20:03:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.070453 13849 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 7 FOLLOWER]: Leader pre-election won for term 8
20:03:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.070691 13849 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 12f7da6233f74ebd81a5cab80b3b3648)
20:03:48.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.070924 13849 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 7 FOLLOWER]: Advancing to term 8
20:03:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.075301 13849 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:48.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.077090 13849 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 8 election: Requested vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.078168 13753 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b"
20:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.078650 13753 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 6 FOLLOWER]: Advancing to term 8
20:03:48.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.081038 13340 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111)
20:03:48.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.087936 13753 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 8.
20:03:48.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.089094 13338 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b, a32a06b806dc4e41b224f010fdedb7b7; no voters: 12f7da6233f74ebd81a5cab80b3b3648
20:03:48.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.089807 13849 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 8 FOLLOWER]: Leader election won for term 8
20:03:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.091030 13849 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 8 LEADER]: Becoming Leader. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Running, Role: LEADER
20:03:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.093560 13849 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.105274 11018 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 reported cstate change: term changed from 7 to 8, leader changed from 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67) to a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66). New cstate: current_term: 8 leader_uuid: "a32a06b806dc4e41b224f010fdedb7b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:03:48.257 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:48.258 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:48.258 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1615), Traces: [0ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [2ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [2ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [14ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [15ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [17ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [17ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [34ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [35ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [36ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [36ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [53ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [54ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [56ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [56ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [74ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [75ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [76ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [76ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [110ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [114ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [116ms] sending RPC to server 12f7da6233f74ebd81a5cab80b3b3648, [118ms] received response from server 12f7da6233f74ebd81a5cab80b3b3648: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [118ms] delaying RPC due to: Network error: Failed to connect to peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831): Connection refused: /127.10.113.67:37831, [1610ms] refreshing cache from master, [1611ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1614ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@51644892(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@7a634c8a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152) -> (continuation of Deferred@389386297 after retry RPC@1454117993) -> (continuation of Deferred@1862077502 after retry RPC@1489261268) -> (continuation of Deferred@1426442922 after retry RPC@791502892), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7b964c06 -> wakeup thread Time-limited test -> (continuation of Deferred@1006611557 after retry RPC@1925198176) -> (continuation of Deferred@565786655 after retry RPC@422252152) -> (continuation of Deferred@389386297 after retry RPC@1454117993) -> (continuation of Deferred@1862077502 after retry RPC@1489261268) -> (continuation of Deferred@1426442922 after retry RPC@791502892))) after lookup
20:03:48.259 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:48.260 [DEBUG - kudu-nio-6] (Connection.java:757) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] Successfully connected to peer
20:03:48.292 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0x3ce7aab3, L:/127.0.0.1:40156 - R:/127.10.113.66:34377] using TOKEN/null
20:03:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.314535 13753 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 8 FOLLOWER]: Refusing update from remote peer a32a06b806dc4e41b224f010fdedb7b7: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
20:03:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.316380 13861 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.001s
20:03:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.317052 13340 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Couldn't send request to peer 12f7da6233f74ebd81a5cab80b3b3648. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:48.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.331696 13887 mvcc.cc:204] Tried to move back new op lower bound from 7317416444807622656 to 7317416443831066624. Current Snapshot: MvccSnapshot[applied={T|T < 7317416444807622656}]
20:03:48.380 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.67:37831
20:03:48.380 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.382091 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:37831
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32785
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.728866 13895 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.729336 13895 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.729779 13895 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.766562 13895 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:48.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.767944 13895 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.67
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.820029 13895 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/logs
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.67:37831
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.67
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32785
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.67
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.821542 13895 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:48.823855 13895 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.844496 13903 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:49.700894 13821 debug-util.cc:398] Leaking SignalData structure 0x7b08000c5080 after lost signal to thread 13609
20:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:49.702564 13821 debug-util.cc:398] Leaking SignalData structure 0x7b08000c9000 after lost signal to thread 13828
20:03:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:50.244915 13901 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 13895
20:03:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:50.286855 13895 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.445s	user 0.549s	sys 0.887s
20:03:50.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:50.287914 13895 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.446s	user 0.550s	sys 0.887s
20:03:50.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:48.845381 13905 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:50.296990 13926 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.297052 13895 server_base.cc:1061] running on GCE node
20:03:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.298502 13895 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:50.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.301265 13895 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.302703 13895 hybrid_clock.cc:648] HybridClock initialized: now 1786478630302655 us; error 48 us; skew 500 ppm
20:03:50.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.303514 13895 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.312067 13895 webserver.cc:533] Webserver started at http://127.10.113.67:32785/ using document root <none> and password file <none>
20:03:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.313026 13895 fs_manager.cc:362] Metadata directory not provided
20:03:50.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.313220 13895 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.322000 13895 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.002s
20:03:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.327381 13931 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:50.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.328682 13895 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.003s
20:03:50.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.328974 13895 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:11 on dist-test-slave-04bb"
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.330888 13895 fs_report.cc:389] FS layout report
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/wal
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/data
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:50.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.398933 13895 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.401309 13895 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.401943 13895 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.405987 13895 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.414786 13941 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.423497 13895 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.423834 13895 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
20:03:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.424836 13895 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.429931 13895 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.430410 13895 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.006s	sys 0.000s
20:03:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.430574 13941 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap starting.
20:03:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.530759 13941 log.cc:826] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Log is configured to *not* fsync() on all Append() calls
20:03:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.654192 13941 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:50.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.655272 13941 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Bootstrap complete.
20:03:50.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.657182 13941 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent bootstrapping tablet: real 0.227s	user 0.174s	sys 0.044s
20:03:50.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.694020 13941 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:50.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.697813 13941 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Initialized, Role: FOLLOWER
20:03:50.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.699026 13941 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.703864 13941 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648: Time spent starting tablet: real 0.046s	user 0.026s	sys 0.004s
20:03:50.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:50.712411 13340 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831): Couldn't send request to peer 12f7da6233f74ebd81a5cab80b3b3648. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.67:37831: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:50.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.722797 13895 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.67:37831
20:03:50.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.723795 14131 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.67:37831 every 8 connection(s)
20:03:50.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.725989 13895 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-2/data/info.pb
20:03:50.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.736017 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 13895
20:03:50.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:50.799 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:50.800 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.66:34377
20:03:50.805 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.806069 14132 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.806324 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 13304
20:03:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.806743 14132 heartbeater.cc:461] Registering TS with master...
20:03:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.808403 14132 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.823798 11017 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.827155 11017 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.67:55333
20:03:50.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.862068 14134 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:50.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.862519 14134 heartbeater.cc:461] Registering TS with master...
20:03:50.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.863260 14134 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:50.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.872074 10887 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.875768 14137 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.876186 14137 heartbeater.cc:461] Registering TS with master...
20:03:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.876966 14137 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:50.880856 10765 ts_manager.cc:194] Re-registered known tserver with Master: 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831)
20:03:50.903 [DEBUG - kudu-nio-6] (Connection.java:239) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] handling channelInactive
20:03:50.904 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state READY due to: connection closed
20:03:50.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:50.906 [DEBUG - Time-limited test] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:50.907 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.907 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.907 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.917 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:50.917 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:50.917 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [12ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@741623206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@685c7c0c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6691c741 -> wakeup thread Time-limited test)) after lookup
20:03:50.918 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:50.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.919 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.937 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:50.938 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:50.938 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [12ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [14ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [14ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@741623206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@685c7c0c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6691c741 -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865))) after lookup
20:03:50.938 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:50.940 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.940 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.940 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.957 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:50.957 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:50.958 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=52), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [12ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [14ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [14ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [35ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [35ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [52ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@741623206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@685c7c0c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6691c741 -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113))) after lookup
20:03:50.958 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:50.960 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.960 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:50.960 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:51.077 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:51.078 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:51.078 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=172), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [12ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [14ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [14ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [35ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [35ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [52ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [53ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [55ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [55ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [168ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [172ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@741623206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@685c7c0c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113) -> (continuation of Deferred@164411331 after retry RPC@1824129751), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6691c741 -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113) -> (continuation of Deferred@164411331 after retry RPC@1824129751))) after lookup
20:03:51.078 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:51.080 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:51.080 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:51.080 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:51.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:51.418 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:51.418 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=512), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [12ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [14ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [14ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [35ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [35ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [52ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [53ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [55ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [55ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [168ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [172ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [173ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [175ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [175ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [508ms] refreshing cache from master, [509ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [512ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@741623206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@685c7c0c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113) -> (continuation of Deferred@164411331 after retry RPC@1824129751) -> (continuation of Deferred@465000293 after retry RPC@656888446), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6691c741 -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113) -> (continuation of Deferred@164411331 after retry RPC@1824129751) -> (continuation of Deferred@465000293 after retry RPC@656888446))) after lookup
20:03:51.419 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:51.420 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:51.420 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:51.420 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:03:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:51.835716 14132 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.010717 14157 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 8 FOLLOWER]: Starting pre-election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:03:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.011217 14157 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:52.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.013971 14157 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:52.029464 13646 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:52.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:52.033988 13646 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.038836 14047 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648" is_pre_election: true
20:03:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.039489 14047 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6719b8a2cff04c88afd580790d9c389b in term 7.
20:03:52.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.040876 13650 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:03:52.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.041538 14157 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 8 FOLLOWER]: Leader pre-election won for term 9
20:03:52.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.041862 14157 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 8 FOLLOWER]: Starting leader election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:03:52.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.042107 14157 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 8 FOLLOWER]: Advancing to term 9
20:03:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.046715 14157 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:52.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.049098 14157 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 9 election: Requested vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:52.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.049211 14047 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:52.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.049909 14047 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 7 FOLLOWER]: Advancing to term 9
20:03:52.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:52.052273 13646 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:03:52.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.061949 14047 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6719b8a2cff04c88afd580790d9c389b in term 9.
20:03:52.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.063216 13650 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:03:52.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.063907 14157 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Leader election won for term 9
20:03:52.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.066254 14157 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 LEADER]: Becoming Leader. State: Replica: 6719b8a2cff04c88afd580790d9c389b, State: Running, Role: LEADER
20:03:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.067256 14157 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:52.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.081691 11017 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b reported cstate change: term changed from 8 to 9, leader changed from a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66) to 6719b8a2cff04c88afd580790d9c389b (127.10.113.65). New cstate: current_term: 9 leader_uuid: "6719b8a2cff04c88afd580790d9c389b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: HEALTHY } } }
20:03:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:52.449548 13646 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:52.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.467463 14047 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 9 FOLLOWER]: Refusing update from remote peer 6719b8a2cff04c88afd580790d9c389b: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
20:03:52.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:52.469144 14157 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [LEADER]: Connected to new peer: Peer: permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
20:03:53.218 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:53.219 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:53.219 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2313), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [2ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [12ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [14ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [14ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [32ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [35ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [35ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [48ms] refreshing cache from master, [48ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [52ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [53ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [55ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [55ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [168ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [172ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [173ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [175ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [175ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [508ms] refreshing cache from master, [509ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [512ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [514ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [515ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [515ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [2309ms] refreshing cache from master, [2309ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [2313ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@741623206(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@685c7c0c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113) -> (continuation of Deferred@164411331 after retry RPC@1824129751) -> (continuation of Deferred@465000293 after retry RPC@656888446) -> (continuation of Deferred@1234274034 after retry RPC@1421379868), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6691c741 -> wakeup thread Time-limited test -> (continuation of Deferred@1219399642 after retry RPC@552460865) -> (continuation of Deferred@696819979 after retry RPC@1723380113) -> (continuation of Deferred@164411331 after retry RPC@1824129751) -> (continuation of Deferred@465000293 after retry RPC@656888446) -> (continuation of Deferred@1234274034 after retry RPC@1421379868))) after lookup
20:03:53.220 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:53.221 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] Successfully connected to peer
20:03:53.251 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0x51274352, L:/127.0.0.1:43210 - R:/127.10.113.65:46157] using TOKEN/null
20:03:53.309 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.66:34377
20:03:53.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:53.311311 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:53.634462 14208 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:53.635068 14208 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:53.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:53.635725 14208 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:53.669416 14208 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:53.670640 14208 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.66
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:53.706949 14208 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:53.708913 14208 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:53.710633 14208 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:53.723106 14217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:55.040324 13646 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:55.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:53.725871 14219 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:55.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:55.070345 14220 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:03:55.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:55.076251 14208 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.352s	user 0.004s	sys 0.000s
20:03:55.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:55.076814 14208 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.352s	user 0.008s	sys 0.000s
20:03:55.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:55.077807 14223 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:55.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.077834 14208 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:55.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.079787 14208 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:55.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.083041 14208 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.084633 14208 hybrid_clock.cc:648] HybridClock initialized: now 1786478635084509 us; error 127 us; skew 500 ppm
20:03:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.086050 14208 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.096276 14208 webserver.cc:533] Webserver started at http://127.10.113.66:33205/ using document root <none> and password file <none>
20:03:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.097947 14208 fs_manager.cc:362] Metadata directory not provided
20:03:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.098330 14208 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.111809 14208 fs_manager.cc:714] Time spent opening directory manager: real 0.008s	user 0.010s	sys 0.000s
20:03:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.118818 14241 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:55.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.120568 14208 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.003s
20:03:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.121098 14208 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:03:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:09 on dist-test-slave-04bb"
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.124450 14208 fs_report.cc:389] FS layout report
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/data
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.256249 14208 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.257800 14208 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.258214 14208 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:55.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.260838 14208 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:55.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.267212 14257 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:03:55.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.276022 14208 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:55.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.276228 14208 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
20:03:55.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.276456 14208 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.281147 14208 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.281411 14208 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
20:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.281800 14257 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap starting.
20:03:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.373134 14257 log.cc:826] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Log is configured to *not* fsync() on all Append() calls
20:03:55.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.463742 14257 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:55.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.464804 14257 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap complete.
20:03:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.467147 14257 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent bootstrapping tablet: real 0.186s	user 0.141s	sys 0.039s
20:03:55.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.479379 14208 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.66:34377
20:03:55.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.479480 14474 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.66:34377 every 8 connection(s)
20:03:55.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.483253 14208 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:03:55.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.494155 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 14208
20:03:55.495 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.493650 14257 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.497659 14257 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Initialized, Role: FOLLOWER
20:03:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.498731 14257 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:55.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.531862 14257 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent starting tablet: real 0.064s	user 0.041s	sys 0.015s
20:03:55.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.559476 14385 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 8 FOLLOWER]: Advancing to term 9
20:03:55.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.560770 14475 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:03:55.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.561267 14475 heartbeater.cc:461] Registering TS with master...
20:03:55.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.562736 14475 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:03:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.578330 14476 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:03:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.578748 14476 heartbeater.cc:461] Registering TS with master...
20:03:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.579792 14476 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:03:55.580 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:55.582 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.65:46157
20:03:55.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.582628 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 13607
20:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.615525 14477 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.615931 14477 heartbeater.cc:461] Registering TS with master...
20:03:55.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.617065 14477 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:03:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.641613 11017 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.642577 10765 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.644614 11017 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.66:45345
20:03:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.647729 10886 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:03:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:55.648540 14475 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:03:55.665 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] handling channelInactive
20:03:55.665 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:55.666 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state READY due to: connection closed
20:03:55.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:55.664556 14248 connection.cc:573] server connection from 127.10.113.65:48701 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:03:55.666 [DEBUG - Time-limited test] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:55.667 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.668 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.668 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.677 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:55.678 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:55.678 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [11ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@91864272(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@1519350a -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dfea642 -> wakeup thread Time-limited test)) after lookup
20:03:55.679 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:55.681 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.681 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.681 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.697 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:55.697 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:55.698 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [11ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [15ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@91864272(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@1519350a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dfea642 -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366))) after lookup
20:03:55.698 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:55.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.699 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.738 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:55.738 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:55.738 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [11ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [15ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [32ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [33ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [71ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@91864272(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@1519350a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dfea642 -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219))) after lookup
20:03:55.740 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:55.741 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.741 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.741 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:55.838 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:55.838 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=172), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [11ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [15ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [32ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [33ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [71ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [75ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [167ms] refreshing cache from master, [167ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [171ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@91864272(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@1519350a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dfea642 -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488))) after lookup
20:03:55.839 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:55.840 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.840 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.840 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.978 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:55.979 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:55.979 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=313), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [11ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [15ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [32ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [33ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [71ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [75ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [167ms] refreshing cache from master, [167ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [171ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [172ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [174ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [174ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [308ms] refreshing cache from master, [308ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [312ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@91864272(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@1519350a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488) -> (continuation of Deferred@969953574 after retry RPC@865975192), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dfea642 -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488) -> (continuation of Deferred@969953574 after retry RPC@865975192))) after lookup
20:03:55.980 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:55.982 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.982 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:55.982 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:56.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:56.739143 14507 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 6719b8a2cff04c88afd580790d9c389b)
20:03:56.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:56.739526 14507 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:56.741868 14507 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:56.745420 13932 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:56.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:56.748826 13932 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:56.752354 14385 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "a32a06b806dc4e41b224f010fdedb7b7" is_pre_election: true
20:03:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:56.754102 13935 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648; no voters: 6719b8a2cff04c88afd580790d9c389b, a32a06b806dc4e41b224f010fdedb7b7
20:03:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:56.754704 14507 raft_consensus.cc:2749] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
20:03:56.999 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:56.999 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)[L],a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:03:57.000 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1333), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [11ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [15ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [32ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [33ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [71ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [75ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [167ms] refreshing cache from master, [167ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [171ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [172ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [174ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [174ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [308ms] refreshing cache from master, [308ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [312ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [314ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [316ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [316ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [1328ms] refreshing cache from master, [1328ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1332ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@91864272(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@1519350a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488) -> (continuation of Deferred@969953574 after retry RPC@865975192) -> (continuation of Deferred@646580190 after retry RPC@2023045612), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dfea642 -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488) -> (continuation of Deferred@969953574 after retry RPC@865975192) -> (continuation of Deferred@646580190 after retry RPC@2023045612))) after lookup
20:03:57.000 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] connecting to peer
20:03:57.001 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:57.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] cleaning up while in state CONNECTING due to: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:57.001 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157
20:03:57.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.081730 14516 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 6719b8a2cff04c88afd580790d9c389b)
20:03:57.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.082201 14516 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:57.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.084640 14516 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:57.089572 14244 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:57.094494 14244 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.096838 14047 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648" is_pre_election: true
20:03:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.097329 14047 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 9.
20:03:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.098404 14246 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7; no voters: 6719b8a2cff04c88afd580790d9c389b
20:03:57.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.099056 14516 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 9 FOLLOWER]: Leader pre-election won for term 10
20:03:57.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.099346 14516 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 6719b8a2cff04c88afd580790d9c389b)
20:03:57.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.099680 14516 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 9 FOLLOWER]: Advancing to term 10
20:03:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.104224 14516 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.105774 14516 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 10 election: Requested vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:03:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.106628 14047 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "a32a06b806dc4e41b224f010fdedb7b7" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648"
20:03:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.107064 14047 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 9 FOLLOWER]: Advancing to term 10
20:03:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:57.109274 14244 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111)
20:03:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.112174 14047 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a32a06b806dc4e41b224f010fdedb7b7 in term 10.
20:03:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.113051 14246 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7; no voters: 6719b8a2cff04c88afd580790d9c389b
20:03:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.113655 14516 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 10 FOLLOWER]: Leader election won for term 10
20:03:57.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.115166 14516 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 10 LEADER]: Becoming Leader. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Running, Role: LEADER
20:03:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.115936 14516 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:03:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.124938 11017 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 reported cstate change: term changed from 9 to 10, leader changed from 6719b8a2cff04c88afd580790d9c389b (127.10.113.65) to a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66). New cstate: current_term: 10 leader_uuid: "a32a06b806dc4e41b224f010fdedb7b7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:03:57.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.630998 14047 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 10 FOLLOWER]: Refusing update from remote peer a32a06b806dc4e41b224f010fdedb7b7: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
20:03:57.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:57.632884 14549 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 36, Time since last communication: 0.000s
20:03:57.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:57.669778 14244 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Couldn't send request to peer 6719b8a2cff04c88afd580790d9c389b. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:58.580 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:03:58.581 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:03:58.581 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=2915), Traces: [0ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [2ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [11ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [13ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [15ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [15ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [27ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [32ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [33ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [33ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [67ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [71ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [75ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [75ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [167ms] refreshing cache from master, [167ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [171ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [172ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [174ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [174ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [308ms] refreshing cache from master, [308ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [312ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [314ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [316ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [316ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [1328ms] refreshing cache from master, [1328ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1332ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [1334ms] sending RPC to server 6719b8a2cff04c88afd580790d9c389b, [1335ms] received response from server 6719b8a2cff04c88afd580790d9c389b: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [1335ms] delaying RPC due to: Network error: Failed to connect to peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157): Connection refused: /127.10.113.65:46157, [2908ms] refreshing cache from master, [2908ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [2914ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@91864272(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@1519350a -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488) -> (continuation of Deferred@969953574 after retry RPC@865975192) -> (continuation of Deferred@646580190 after retry RPC@2023045612) -> (continuation of Deferred@635479332 after retry RPC@1953215021), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dfea642 -> wakeup thread Time-limited test -> (continuation of Deferred@695196271 after retry RPC@1912317366) -> (continuation of Deferred@1402001594 after retry RPC@2146042219) -> (continuation of Deferred@1537598950 after retry RPC@1953187488) -> (continuation of Deferred@969953574 after retry RPC@865975192) -> (continuation of Deferred@646580190 after retry RPC@2023045612) -> (continuation of Deferred@635479332 after retry RPC@1953215021))) after lookup
20:03:58.582 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:03:58.584 [DEBUG - kudu-nio-5] (Connection.java:757) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] Successfully connected to peer
20:03:58.611 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0x873e2363, L:/127.0.0.1:44536 - R:/127.10.113.66:34377] using TOKEN/null
20:03:58.706 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.65:46157
20:03:58.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:58.711422 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:46157
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42403
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:59.370888 14591 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:59.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:59.371949 14591 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:59.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:59.373047 14591 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:59.429747 14591 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:59.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:59.431506 14591 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.65
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:59.473994 14591 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/logs
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.65:46157
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.65
20:03:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42403
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.65
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:59.475508 14591 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:03:59.477676 14591 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:03:59.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:59.493593 14608 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:00.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:00.317041 14244 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 -> Peer 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157): Couldn't send request to peer 6719b8a2cff04c88afd580790d9c389b. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.65:46157: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:00.896466 14601 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 14591
20:04:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:01.400700 14591 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.907s	user 0.682s	sys 1.142s
20:04:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:01.401067 14591 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.907s	user 0.682s	sys 1.142s
20:04:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:03:59.496657 14609 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:01.403445 14611 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:01.406927 14610 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1908 milliseconds
20:04:01.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.407035 14591 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:01.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.408413 14591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.410907 14591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.412329 14591 hybrid_clock.cc:648] HybridClock initialized: now 1786478641412276 us; error 47 us; skew 500 ppm
20:04:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.413220 14591 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:01.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.420429 14591 webserver.cc:533] Webserver started at http://127.10.113.65:42403/ using document root <none> and password file <none>
20:04:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.421536 14591 fs_manager.cc:362] Metadata directory not provided
20:04:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.421826 14591 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:01.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.430657 14591 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
20:04:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.435464 14635 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.436580 14591 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.000s
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.436971 14591 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6719b8a2cff04c88afd580790d9c389b"
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:06 on dist-test-slave-04bb"
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.439103 14591 fs_report.cc:389] FS layout report
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/wal
20:04:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/data
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.505754 14591 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:01.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.507467 14591 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.508054 14591 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.511631 14591 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.518800 14643 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:04:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.530761 14591 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.531018 14591 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.014s	user 0.002s	sys 0.000s
20:04:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.531421 14591 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.536732 14591 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.536954 14591 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.006s	sys 0.000s
20:04:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.537420 14643 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap starting.
20:04:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.627216 14643 log.cc:826] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Log is configured to *not* fsync() on all Append() calls
20:04:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.753640 14643 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.754768 14643 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Bootstrap complete.
20:04:01.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.756907 14643 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent bootstrapping tablet: real 0.220s	user 0.162s	sys 0.051s
20:04:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.779858 14643 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.782842 14643 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6719b8a2cff04c88afd580790d9c389b, State: Initialized, Role: FOLLOWER
20:04:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.783900 14643 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.789256 14643 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b: Time spent starting tablet: real 0.032s	user 0.022s	sys 0.008s
20:04:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.807832 14591 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.65:46157
20:04:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.808219 14819 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.65:46157 every 8 connection(s)
20:04:01.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.811012 14591 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-0/data/info.pb
20:04:01.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.817131 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 14591
20:04:01.862 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:01.862 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.10.113.66:34377
20:04:01.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.863663 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 14208
20:04:01.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.891959 14821 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:04:01.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.892491 14821 heartbeater.cc:461] Registering TS with master...
20:04:01.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.894143 14821 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:04:01.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.897840 14822 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:04:01.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.898197 14822 heartbeater.cc:461] Registering TS with master...
20:04:01.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.899155 14822 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:04:01.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.900278 11018 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:04:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.902684 14825 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:04:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.903031 14825 heartbeater.cc:461] Registering TS with master...
20:04:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.903462 10887 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.903769 11018 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.65:37123
20:04:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.903944 14825 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:04:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:01.908888 10767 ts_manager.cc:194] Re-registered known tserver with Master: 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:04:01.923 [DEBUG - kudu-nio-5] (Connection.java:239) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] handling channelInactive
20:04:01.923 [DEBUG - kudu-nio-5] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state READY due to: connection closed
20:04:01.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:01.925 [DEBUG - Time-limited test] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:04:01.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:01.925488 14839 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:01.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.874865 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.10.113.66:57779 (local address 127.10.113.65:46157)
20:04:01.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.881419 (+  6554us) server_negotiation.cc:207] Beginning negotiation
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.881451 (+    32us) server_negotiation.cc:400] Waiting for connection header
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.881518 (+    67us) server_negotiation.cc:408] Connection header received
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.881733 (+   215us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.881777 (+    44us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.882215 (+   438us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.882488 (+   273us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:04:01.923603 (+ 41115us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.10.113.66:57779: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:04:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5807,"thread_start_us":5281,"threads_started":1}
20:04:01.926 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.926 [DEBUG - kudu-nio-6] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.926 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.938 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:04:01.938 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:04:01.938 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [13ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@612739955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@d569f14 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@167c1c3 -> wakeup thread Time-limited test)) after lookup
20:04:01.939 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:04:01.940 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.940 [DEBUG - kudu-nio-7] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.940 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.957 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:04:01.957 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:04:01.957 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [13ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [14ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [15ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [15ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@612739955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@d569f14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@167c1c3 -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254))) after lookup
20:04:01.958 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:04:01.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.959 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:01.998 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:04:01.998 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:04:01.998 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [13ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [14ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [15ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [15ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [34ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [34ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [72ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@612739955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@d569f14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@167c1c3 -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740))) after lookup
20:04:01.999 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:04:02.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.000 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.058 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:04:02.058 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:04:02.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=134), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [13ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [14ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [15ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [15ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [34ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [34ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [72ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [75ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [75ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [132ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@612739955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@d569f14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@167c1c3 -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135))) after lookup
20:04:02.060 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:04:02.061 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.061 [DEBUG - kudu-nio-2] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.061 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.578 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:04:02.578 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:04:02.578 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=653), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [13ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [14ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [15ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [15ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [34ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [34ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [72ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [75ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [75ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [132ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [134ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [136ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [136ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [649ms] refreshing cache from master, [649ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [652ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@612739955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@d569f14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135) -> (continuation of Deferred@170943869 after retry RPC@1035489498), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@167c1c3 -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135) -> (continuation of Deferred@170943869 after retry RPC@1035489498))) after lookup
20:04:02.579 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:04:02.580 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.580 [DEBUG - kudu-nio-3] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.580 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:02.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:02.907861 14821 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:04:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.022500 14844 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:03.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.023049 14844 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:03.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.025597 14844 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67:37831), a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:04:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.032698 14637 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:04:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.036908 14637 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:04:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.039868 14047 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "6719b8a2cff04c88afd580790d9c389b" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "12f7da6233f74ebd81a5cab80b3b3648" is_pre_election: true
20:04:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.041654 14638 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6719b8a2cff04c88afd580790d9c389b; no voters: 12f7da6233f74ebd81a5cab80b3b3648, a32a06b806dc4e41b224f010fdedb7b7
20:04:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.042379 14844 raft_consensus.cc:2749] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Leader pre-election lost for term 10. Reason: could not achieve majority
20:04:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.163933 11068 debug-util.cc:398] Leaking SignalData structure 0x7b080007b820 after lost signal to thread 10956
20:04:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.164642 11068 debug-util.cc:398] Leaking SignalData structure 0x7b08000987a0 after lost signal to thread 11073
20:04:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.312414 14853 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:04:03.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.312929 14853 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.314839 14853 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:04:03.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.317960 13935 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:04:03.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.321798 13935 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:04:03.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.327395 14746 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b" is_pre_election: true
20:04:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.328047 14746 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 9.
20:04:03.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.329384 13932 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:04:03.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.330065 14853 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 10 FOLLOWER]: Leader pre-election won for term 11
20:04:03.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.330432 14853 raft_consensus.cc:493] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 10 FOLLOWER]: Starting leader election (detected failure of leader a32a06b806dc4e41b224f010fdedb7b7)
20:04:03.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.330758 14853 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 10 FOLLOWER]: Advancing to term 11
20:04:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.336139 14853 raft_consensus.cc:515] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.337743 14853 leader_election.cc:290] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 11 election: Requested vote from peers a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377), 6719b8a2cff04c88afd580790d9c389b (127.10.113.65:46157)
20:04:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.338841 14746 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "242ddce46e42481b9276d573742023bb" candidate_uuid: "12f7da6233f74ebd81a5cab80b3b3648" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "6719b8a2cff04c88afd580790d9c389b"
20:04:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.339421 14746 raft_consensus.cc:3060] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 9 FOLLOWER]: Advancing to term 11
20:04:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.341279 13935 leader_election.cc:336] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111)
20:04:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.346401 14746 raft_consensus.cc:2468] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 12f7da6233f74ebd81a5cab80b3b3648 in term 11.
20:04:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.347245 13932 leader_election.cc:304] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 12f7da6233f74ebd81a5cab80b3b3648, 6719b8a2cff04c88afd580790d9c389b; no voters: a32a06b806dc4e41b224f010fdedb7b7
20:04:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.347800 14853 raft_consensus.cc:2804] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 11 FOLLOWER]: Leader election won for term 11
20:04:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.349540 14853 raft_consensus.cc:697] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [term 11 LEADER]: Becoming Leader. State: Replica: 12f7da6233f74ebd81a5cab80b3b3648, State: Running, Role: LEADER
20:04:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.350593 14853 consensus_queue.cc:237] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.359843 11018 catalog_manager.cc:5719] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 reported cstate change: term changed from 10 to 11, leader changed from a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66) to 12f7da6233f74ebd81a5cab80b3b3648 (127.10.113.67). New cstate: current_term: 11 leader_uuid: "12f7da6233f74ebd81a5cab80b3b3648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } health_report { overall_health: UNKNOWN } } }
20:04:03.378 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:04:03.379 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831),6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)[L]]}]
20:04:03.379 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1454), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [13ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [14ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [15ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [15ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [34ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [34ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [72ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [75ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [75ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [132ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [134ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [136ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [136ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [649ms] refreshing cache from master, [649ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [652ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [654ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [655ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [655ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1449ms] refreshing cache from master, [1449ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1453ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@612739955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@d569f14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135) -> (continuation of Deferred@170943869 after retry RPC@1035489498) -> (continuation of Deferred@2119654609 after retry RPC@18279745), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@167c1c3 -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135) -> (continuation of Deferred@170943869 after retry RPC@1035489498) -> (continuation of Deferred@2119654609 after retry RPC@18279745))) after lookup
20:04:03.380 [DEBUG - kudu-nio-2] (Connection.java:748) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] connecting to peer
20:04:03.381 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:03.382 [DEBUG - kudu-nio-4] (Connection.java:709) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] cleaning up while in state CONNECTING due to: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:03.382 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377) for tablet 242ddce46e42481b9276d573742023bb: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377
20:04:03.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:03.770254 13935 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.918205 14746 raft_consensus.cc:1275] T 242ddce46e42481b9276d573742023bb P 6719b8a2cff04c88afd580790d9c389b [term 11 FOLLOWER]: Refusing update from remote peer 12f7da6233f74ebd81a5cab80b3b3648: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
20:04:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:03.919692 14853 consensus_queue.cc:1048] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
20:04:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:06.195466 13935 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:07.478 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 242ddce46e42481b9276d573742023bb for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674' with partition [<start>, <end>)
20:04:07.478 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=242ddce46e42481b9276d573742023bb@[12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)[L],6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157),a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)]}]
20:04:07.478 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=242ddce46e42481b9276d573742023bb, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=5553), Traces: [0ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [13ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [14ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [15ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [15ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [31ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [33ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [34ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [34ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [72ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [73ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [75ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [75ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [129ms] refreshing cache from master, [129ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [132ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [134ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [136ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [136ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [649ms] refreshing cache from master, [649ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [652ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [654ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [655ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [655ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1449ms] refreshing cache from master, [1449ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [1453ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, [1455ms] sending RPC to server a32a06b806dc4e41b224f010fdedb7b7, [1457ms] received response from server a32a06b806dc4e41b224f010fdedb7b7: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [1457ms] delaying RPC due to: Network error: Failed to connect to peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377): Connection refused: /127.10.113.66:34377, [5549ms] refreshing cache from master, [5549ms] Sub RPC GetTableLocations: sending RPC to server master-127.10.113.124:42269, [5552ms] Sub RPC GetTableLocations: received response from server master-127.10.113.124:42269: OK, deferred=Deferred@612739955(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$232/0x00007fd2c4338f98@d569f14 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135) -> (continuation of Deferred@170943869 after retry RPC@1035489498) -> (continuation of Deferred@2119654609 after retry RPC@18279745) -> (continuation of Deferred@1522386011 after retry RPC@1819756200), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@167c1c3 -> wakeup thread Time-limited test -> (continuation of Deferred@1376970110 after retry RPC@1833193254) -> (continuation of Deferred@508627151 after retry RPC@1369659740) -> (continuation of Deferred@1668993045 after retry RPC@676863135) -> (continuation of Deferred@170943869 after retry RPC@1035489498) -> (continuation of Deferred@2119654609 after retry RPC@18279745) -> (continuation of Deferred@1522386011 after retry RPC@1819756200))) after lookup
20:04:07.479 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] connecting to peer
20:04:07.480 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] Successfully connected to peer
20:04:07.499 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0x5309dbbe, L:/127.0.0.1:60864 - R:/127.10.113.67:37831] using TOKEN/null
20:04:07.543 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.10.113.66:34377
20:04:07.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:07.545889 10693 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.884217 14916 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.884784 14916 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:07.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.885262 14916 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:07.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.917995 14916 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:07.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.918798 14916 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.113.66
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:07.958071 14916 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/logs
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.113.84:42909
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.113.66:34377
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.113.66
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33205
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.113.126:45721,127.10.113.125:33831,127.10.113.124:42269
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.113.66
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 12dab910780628d7c8d59e0ab03bf2fa661d310d
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 19:45:48 UTC on e7f111948823
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13295
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:07.959318 14916 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:07.961012 14916 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:04:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.976398 14926 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:08.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:08.299784 11068 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9600 after lost signal to thread 10956
20:04:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:08.300771 11068 debug-util.cc:398] Leaking SignalData structure 0x7b08000b44a0 after lost signal to thread 11073
20:04:08.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:08.677768 13935 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111) [suppressed 11 similar messages]
20:04:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:08.689972 13935 consensus_peers.cc:597] T 242ddce46e42481b9276d573742023bb P 12f7da6233f74ebd81a5cab80b3b3648 -> Peer a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377): Couldn't send request to peer a32a06b806dc4e41b224f010fdedb7b7. Status: Network error: Client connection negotiation failed: client connection to 127.10.113.66:34377: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:04:09.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.981233 14929 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:07.977629 14927 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:04:09.317684 14928 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1337 milliseconds
20:04:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.317804 14916 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:09.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.319105 14916 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.321403 14916 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.322906 14916 hybrid_clock.cc:648] HybridClock initialized: now 1786478649322861 us; error 65 us; skew 500 ppm
20:04:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.324147 14916 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:09.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.331130 14916 webserver.cc:533] Webserver started at http://127.10.113.66:33205/ using document root <none> and password file <none>
20:04:09.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.332269 14916 fs_manager.cc:362] Metadata directory not provided
20:04:09.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.332477 14916 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.341917 14916 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.001s
20:04:09.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.346871 14955 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.348021 14916 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
20:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.348390 14916 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data,/tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32a06b806dc4e41b224f010fdedb7b7"
20:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:03:09 on dist-test-slave-04bb"
20:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.350353 14916 fs_report.cc:389] FS layout report
20:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/wal
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/data
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.412179 14916 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:09.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.414008 14916 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:09.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.414639 14916 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:09.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.417783 14916 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.424392 14963 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
20:04:09.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.433688 14916 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:09.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.433917 14916 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.002s	sys 0.001s
20:04:09.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.434139 14916 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:09.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.438874 14916 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:09.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.439143 14916 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
20:04:09.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.439540 14963 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap starting.
20:04:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.544924 14963 log.cc:826] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Log is configured to *not* fsync() on all Append() calls
20:04:09.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.614809 14916 rpc_server.cc:307] RPC server started. Bound to: 127.10.113.66:34377
20:04:09.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.615173 15161 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.113.66:34377 every 8 connection(s)
20:04:09.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.618623 14916 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskK9yTow/test-tmp/mini-kudu-cluster7754654634351045692/ts-1/data/info.pb
20:04:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.624109 10693 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu as pid 14916
20:04:09.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:09.673 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:09.691 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786478592674, tablet=242ddce46e42481b9276d573742023bb, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:09.697 [DEBUG - main] (Connection.java:575) [peer a32a06b806dc4e41b224f010fdedb7b7(127.10.113.66:34377)] disconnecting while in state TERMINATED
20:04:09.698 [DEBUG - main] (Connection.java:575) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] disconnecting while in state READY
20:04:09.698 [DEBUG - main] (Connection.java:575) [peer master-127.10.113.124:42269(127.10.113.124:42269)] disconnecting while in state READY
20:04:09.699 [DEBUG - main] (Connection.java:575) [peer 6719b8a2cff04c88afd580790d9c389b(127.10.113.65:46157)] disconnecting while in state TERMINATED
20:04:09.699 [DEBUG - main] (Connection.java:575) [peer master-127.10.113.126:45721(127.10.113.126:45721)] disconnecting while in state READY
20:04:09.699 [DEBUG - main] (Connection.java:575) [peer master-127.10.113.125:33831(127.10.113.125:33831)] disconnecting while in state READY
20:04:09.702 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.113.125:33831(127.10.113.125:33831)] handling channelInactive
20:04:09.702 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.113.125:33831(127.10.113.125:33831)] cleaning up while in state READY due to: connection closed
20:04:09.707 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] handling channelInactive
20:04:09.707 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 12f7da6233f74ebd81a5cab80b3b3648(127.10.113.67:37831)] cleaning up while in state READY due to: connection closed
20:04:09.708 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.10.113.124:42269(127.10.113.124:42269)] handling channelInactive
20:04:09.708 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.10.113.124:42269(127.10.113.124:42269)] cleaning up while in state READY due to: connection closed
20:04:09.708 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:09.710 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.113.126:45721(127.10.113.126:45721)] handling channelInactive
20:04:09.710 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.113.126:45721(127.10.113.126:45721)] cleaning up while in state READY due to: connection closed
20:04:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.710608 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 14591
20:04:09.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.757480 14963 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:09.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.758632 14963 tablet_bootstrap.cc:492] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Bootstrap complete.
20:04:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.760557 14963 ts_tablet_manager.cc:1403] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent bootstrapping tablet: real 0.321s	user 0.213s	sys 0.043s
20:04:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.770210 15168 heartbeater.cc:344] Connected to a master server at 127.10.113.125:33831
20:04:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.770725 15168 heartbeater.cc:461] Registering TS with master...
20:04:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.772109 15168 heartbeater.cc:507] Master 127.10.113.125:33831 requested a full tablet report, sending...
20:04:09.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.777395 10887 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:04:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.783434 15178 heartbeater.cc:344] Connected to a master server at 127.10.113.126:45721
20:04:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.783478 15166 heartbeater.cc:344] Connected to a master server at 127.10.113.124:42269
20:04:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.783895 15178 heartbeater.cc:461] Registering TS with master...
20:04:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.783941 15166 heartbeater.cc:461] Registering TS with master...
20:04:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.785099 15178 heartbeater.cc:507] Master 127.10.113.126:45721 requested a full tablet report, sending...
20:04:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.788932 15166 heartbeater.cc:507] Master 127.10.113.124:42269 requested a full tablet report, sending...
20:04:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.792392 11017 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:04:09.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.799362 11017 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.113.66:38403
20:04:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.800055 10767 ts_manager.cc:194] Re-registered known tserver with Master: a32a06b806dc4e41b224f010fdedb7b7 (127.10.113.66:34377)
20:04:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.805807 14963 raft_consensus.cc:359] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.808904 14963 raft_consensus.cc:740] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32a06b806dc4e41b224f010fdedb7b7, State: Initialized, Role: FOLLOWER
20:04:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.810031 14963 consensus_queue.cc:260] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12f7da6233f74ebd81a5cab80b3b3648" member_type: VOTER last_known_addr { host: "127.10.113.67" port: 37831 } } peers { permanent_uuid: "a32a06b806dc4e41b224f010fdedb7b7" member_type: VOTER last_known_addr { host: "127.10.113.66" port: 34377 } } peers { permanent_uuid: "6719b8a2cff04c88afd580790d9c389b" member_type: VOTER last_known_addr { host: "127.10.113.65" port: 46157 } }
20:04:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.820300 15166 heartbeater.cc:499] Master 127.10.113.124:42269 was elected leader, sending a full tablet report...
20:04:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.824864 14963 ts_tablet_manager.cc:1434] T 242ddce46e42481b9276d573742023bb P a32a06b806dc4e41b224f010fdedb7b7: Time spent starting tablet: real 0.064s	user 0.028s	sys 0.004s
20:04:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.837941 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 14916
20:04:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.888036 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 13895
20:04:09.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.930469 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 10709
20:04:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:09.981626 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 10819
20:04:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:04:10.025264 10693 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskK9yTow/build/tsan/bin/kudu with pid 10942
20:04:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:04:11Z chronyd exiting

Time: 77.527

OK (1 test)

