JUnit version 4.13.2
.06:12:28.517 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1786515148509
06:12:28.587 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
06:12:28.626 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:12:28.641 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0kB5X1/test-tmp
06:12:28.665 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:12:28.668 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0kB5X1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:12:28.847 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

06:12:29.037 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:12:29.038 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

06:12:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.046790  6302 mini_oidc.cc:150] Starting JWKS server
06:12:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.051420  6302 webserver.cc:533] Webserver started at http://127.0.0.1:41963/ using document root <none> and password file <none>
06:12:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.052057  6302 mini_oidc.cc:186] Starting OIDC Discovery server
06:12:29.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.054338  6302 webserver.cc:533] Webserver started at http://127.0.0.1:44403/ using document root <none> and password file <none>
06:12:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T06:12:29Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
06:12:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T06:12:29Z Disabled control of system clock
06:12:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.114379  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/wal
06:12:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data
06:12:29.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:29.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:29.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/logs
06:12:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data/info.pb
06:12:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.190:40895
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.190
06:12:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:12:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:12:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41963/jwks.json with env {}
06:12:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.408277  6315 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.408860  6315 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:29.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.409329  6315 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.440876  6315 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
06:12:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.441191  6315 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.441394  6315 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
06:12:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.441690  6315 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
06:12:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.442015  6315 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
06:12:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.442256  6315 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41963/jwks.json
06:12:29.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.477900  6315 master_runner.cc:387] Master server non-default flags:
06:12:29.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data
06:12:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/wal
06:12:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:12:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.190:40895
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41963/jwks.json
06:12:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data/info.pb
06:12:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.190
06:12:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/logs
06:12:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.479398  6315 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:29.481266  6315 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.492309  6322 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:31.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:30.896941  6321 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 6315
06:12:31.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:29.493595  6323 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:31.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.039578  6315 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.546s	user 0.653s	sys 0.892s
06:12:31.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.041250  6315 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.548s	user 0.654s	sys 0.892s
06:12:31.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.041890  6325 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.044682  6324 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1546 milliseconds
06:12:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.044751  6315 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.046185  6315 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.048693  6315 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.050029  6315 hybrid_clock.cc:648] HybridClock initialized: now 1786515151050025 us; error 89 us; skew 500 ppm
06:12:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.050906  6315 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:31.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.056931  6315 webserver.cc:533] Webserver started at http://127.6.39.190:39455/ using document root <none> and password file <none>
06:12:31.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.057997  6315 fs_manager.cc:362] Metadata directory not provided
06:12:31.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.058406  6315 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:31.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.059079  6315 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
06:12:31.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.063490  6315 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data/instance:
06:12:31.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86dff63507b54089a717b2f1b12b249b"
06:12:31.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:31 on dist-test-slave-vq2q"
06:12:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.064865  6315 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/wal/instance:
06:12:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86dff63507b54089a717b2f1b12b249b"
06:12:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:31 on dist-test-slave-vq2q"
06:12:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.072296  6315 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.004s
06:12:31.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.078087  6334 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.079170  6315 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
06:12:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.079521  6315 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/wal
06:12:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86dff63507b54089a717b2f1b12b249b"
06:12:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:31 on dist-test-slave-vq2q"
06:12:31.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.079927  6315 fs_report.cc:389] FS layout report
06:12:31.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:31.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/wal
06:12:31.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/wal
06:12:31.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data/data
06:12:31.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:31.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:31.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:31.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:31.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.129076  6315 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.130872  6315 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.131382  6315 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:31.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.200027  6315 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.190:40895
06:12:31.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.200111  6399 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.190:40895 every 8 connection(s)
06:12:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.203004  6315 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/data/info.pb
06:12:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.206610  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 6315
06:12:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.207134  6302 external_mini_cluster.cc:1449] Reading /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-0/wal/instance
06:12:31.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.209991  6400 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:12:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.212015  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/wal
06:12:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data
06:12:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:31.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/logs
06:12:31.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data/info.pb
06:12:31.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:31.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:31.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.189:43971
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.189
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:12:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:31.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:12:31.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41963/jwks.json with env {}
06:12:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.222994  6400 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } has no permanent_uuid. Determining permanent_uuid...
06:12:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.250875  6400 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } has no permanent_uuid. Determining permanent_uuid...
06:12:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.253273  6338 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:31.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.257566  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.189:43971: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.307078  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } attempt: 1
06:12:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.312903  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.189:43971: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.381275  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } attempt: 2
06:12:31.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.385317  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.189:43971: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.476807  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } attempt: 3
06:12:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.481644  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.189:43971: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.515632  6402 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.516172  6402 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.516597  6402 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.546953  6402 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
06:12:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.547223  6402 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.547413  6402 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
06:12:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.547600  6402 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
06:12:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.547821  6402 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
06:12:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.547978  6402 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41963/jwks.json
06:12:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.583549  6402 master_runner.cc:387] Master server non-default flags:
06:12:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data
06:12:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/wal
06:12:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:12:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:12:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.189:43971
06:12:31.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:12:31.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:31.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:12:31.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41963/jwks.json
06:12:31.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:31.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data/info.pb
06:12:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.189
06:12:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/logs
06:12:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:12:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.584861  6402 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.587055  6402 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:31.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.598572  6412 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:31.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.625049  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } attempt: 4
06:12:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.632334  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.189:43971: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:31.931811  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } attempt: 5
06:12:31.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.939786  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.189:43971: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.487331  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } attempt: 6
06:12:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:32.512794  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.189:43971: Network error: Client connection negotiation failed: client connection to 127.6.39.189:43971: connect: Connection refused (error 111)
06:12:32.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:31.599743  6414 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:32.935343  6416 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:32.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:32.938386  6402 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.339s	user 0.595s	sys 0.737s
06:12:32.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:32.938743  6402 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.340s	user 0.595s	sys 0.737s
06:12:32.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:32.940658  6415 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1333 milliseconds
06:12:32.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.940729  6402 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:32.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.942183  6402 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.944725  6402 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.946092  6402 hybrid_clock.cc:648] HybridClock initialized: now 1786515152946004 us; error 86 us; skew 500 ppm
06:12:32.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.946996  6402 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.954315  6402 webserver.cc:533] Webserver started at http://127.6.39.189:45929/ using document root <none> and password file <none>
06:12:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.955384  6402 fs_manager.cc:362] Metadata directory not provided
06:12:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.955610  6402 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.956187  6402 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
06:12:32.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.960724  6402 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data/instance:
06:12:32.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a76e2b12b70451cb9836e7b10afef77"
06:12:32.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:32 on dist-test-slave-vq2q"
06:12:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.962069  6402 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/wal/instance:
06:12:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a76e2b12b70451cb9836e7b10afef77"
06:12:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:32 on dist-test-slave-vq2q"
06:12:32.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.974207  6402 fs_manager.cc:696] Time spent creating directory manager: real 0.011s	user 0.010s	sys 0.002s
06:12:32.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.980895  6425 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:32.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.982218  6402 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
06:12:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.982579  6402 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/wal
06:12:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a76e2b12b70451cb9836e7b10afef77"
06:12:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:32 on dist-test-slave-vq2q"
06:12:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:32.982990  6402 fs_report.cc:389] FS layout report
06:12:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/wal
06:12:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/wal
06:12:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data/data
06:12:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:32.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:32.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:32.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:32.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:32.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:32.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:33.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.048725  6402 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.050397  6402 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.050894  6402 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.121709  6402 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.189:43971
06:12:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.121791  6486 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.189:43971 every 8 connection(s)
06:12:33.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.124747  6402 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/data/info.pb
06:12:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.130903  6487 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:12:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.134102  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 6402
06:12:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.134492  6302 external_mini_cluster.cc:1449] Reading /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-1/wal/instance
06:12:33.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.136507  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/wal
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/logs
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data/info.pb
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.188:40371
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.188
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:33.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:33.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:33.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:12:33.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:33.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:12:33.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41963/jwks.json with env {}
06:12:33.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.143288  6487 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } has no permanent_uuid. Determining permanent_uuid...
06:12:33.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.164706  6487 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } has no permanent_uuid. Determining permanent_uuid...
06:12:33.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.178190  6487 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } has no permanent_uuid. Determining permanent_uuid...
06:12:33.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.180593  6429 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.184619  6487 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.234072  6487 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 1
06:12:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.238229  6487 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.306566  6487 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 2
06:12:33.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.310932  6487 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.402402  6487 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 3
06:12:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.407006  6487 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.440687  6489 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:33.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.441387  6489 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.442099  6489 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:33.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.472994  6489 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
06:12:33.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.473344  6489 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.473673  6489 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
06:12:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.474028  6489 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
06:12:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.474396  6489 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
06:12:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.474676  6489 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41963/jwks.json
06:12:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.510149  6489 master_runner.cc:387] Master server non-default flags:
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/wal
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:12:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:12:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.188:40371
06:12:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:12:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:33.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:12:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41963/jwks.json
06:12:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data/info.pb
06:12:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.188
06:12:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/logs
06:12:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:33.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:33.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.511888  6489 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:33.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.514206  6489 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.525904  6501 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:33.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.550467  6487 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 4
06:12:33.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.555902  6487 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.573298  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } attempt: 7
06:12:33.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.588903  6400 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } has no permanent_uuid. Determining permanent_uuid...
06:12:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.604393  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.662899  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 1
06:12:33.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.668357  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.749840  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 2
06:12:33.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.755522  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.841040  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 3
06:12:33.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.847412  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.855357  6487 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 5
06:12:33.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.860581  6487 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:33.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:33.987890  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 4
06:12:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.993675  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.277197  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 5
06:12:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:34.282987  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.408118  6487 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 6
06:12:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:34.417826  6487 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.529697  6504 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:33.526557  6502 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:34.679929  6503 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1148 milliseconds
06:12:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.680068  6489 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.681394  6489 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:34.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.684423  6489 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:34.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.685786  6489 hybrid_clock.cc:648] HybridClock initialized: now 1786515154685743 us; error 51 us; skew 500 ppm
06:12:34.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.686834  6489 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:34.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.694830  6489 webserver.cc:533] Webserver started at http://127.6.39.188:41011/ using document root <none> and password file <none>
06:12:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.696131  6489 fs_manager.cc:362] Metadata directory not provided
06:12:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.696425  6489 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.697019  6489 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
06:12:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.703465  6489 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data/instance:
06:12:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9145881f908a44b8b2a0a469ad677d6e"
06:12:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:34 on dist-test-slave-vq2q"
06:12:34.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.708544  6489 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/wal/instance:
06:12:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9145881f908a44b8b2a0a469ad677d6e"
06:12:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:34 on dist-test-slave-vq2q"
06:12:34.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.716079  6489 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.000s
06:12:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.721874  6515 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.722986  6489 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
06:12:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.723310  6489 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/wal
06:12:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9145881f908a44b8b2a0a469ad677d6e"
06:12:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:34 on dist-test-slave-vq2q"
06:12:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.723680  6489 fs_report.cc:389] FS layout report
06:12:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/wal
06:12:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/wal
06:12:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data/data
06:12:34.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.786887  6489 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:34.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.788554  6489 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.789072  6489 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.835501  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 6
06:12:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:34.843205  6400 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.6.39.188:40371: Network error: Client connection negotiation failed: client connection to 127.6.39.188:40371: connect: Connection refused (error 111)
06:12:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.863046  6489 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.188:40371
06:12:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.863157  6578 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.188:40371 every 8 connection(s)
06:12:34.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.865971  6489 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/data/info.pb
06:12:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.869453  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 6489
06:12:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.869906  6302 external_mini_cluster.cc:1449] Reading /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/master-2/wal/instance
06:12:34.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.872720  6579 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:12:34.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.886468  6579 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } has no permanent_uuid. Determining permanent_uuid...
06:12:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.909324  6579 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } has no permanent_uuid. Determining permanent_uuid...
06:12:34.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.918660  6579 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } has no permanent_uuid. Determining permanent_uuid...
06:12:34.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.936010  6579 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e: Bootstrap starting.
06:12:34.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.942054  6579 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e: Neither blocks nor log segments found. Creating new log.
06:12:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.944365  6579 log.cc:826] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e: Log is configured to *not* fsync() on all Append() calls
06:12:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.948441  6579 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e: No bootstrap required, opened a new log
06:12:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.967072  6579 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.967679  6579 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:12:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.967882  6579 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9145881f908a44b8b2a0a469ad677d6e, State: Initialized, Role: FOLLOWER
06:12:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.968567  6579 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:34.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.971220  6588 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:34.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.972122  6588 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: This master's current role is: FOLLOWER
06:12:34.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.972352  6579 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: configured and running, proceeding with master startup.
06:12:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:34.998939  6601 catalog_manager.cc:1614] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:12:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:34.999275  6601 catalog_manager.cc:884] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:12:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:34.994458  6489 master_runner.cc:187] Error getting master registration for 127.6.39.190:40895: OK, instance_id { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" instance_seqno: 1786515151181510 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:12:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.017355  6489 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
06:12:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:35.044345  6523 tablet.cc:2406] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:12:35.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.309630  6588 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:12:35.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.310068  6588 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:35.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.312012  6588 leader_election.cc:290] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86dff63507b54089a717b2f1b12b249b (127.6.39.190:40895), 2a76e2b12b70451cb9836e7b10afef77 (127.6.39.189:43971)
06:12:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.478319  6487 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 7
06:12:35.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.503883  6487 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: Bootstrap starting.
06:12:35.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.511623  6487 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: Neither blocks nor log segments found. Creating new log.
06:12:35.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.514015  6487 log.cc:826] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: Log is configured to *not* fsync() on all Append() calls
06:12:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.519129  6487 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: No bootstrap required, opened a new log
06:12:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.548144  6487 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.549014  6487 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:12:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.549386  6487 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a76e2b12b70451cb9836e7b10afef77, State: Initialized, Role: FOLLOWER
06:12:35.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.550690  6487 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.553488  6606 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.554522  6606 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [sys.catalog]: This master's current role is: FOLLOWER
06:12:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.555308  6487 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [sys.catalog]: configured and running, proceeding with master startup.
06:12:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.555989  6457 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9145881f908a44b8b2a0a469ad677d6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a76e2b12b70451cb9836e7b10afef77" is_pre_election: true
06:12:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.557796  6457 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9145881f908a44b8b2a0a469ad677d6e in term 0.
06:12:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:35.559197  6432 tablet.cc:2406] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:12:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.560667  6518 leader_election.cc:304] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a76e2b12b70451cb9836e7b10afef77, 9145881f908a44b8b2a0a469ad677d6e; no voters: 
06:12:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.561539  6588 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Leader pre-election won for term 1
06:12:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.561896  6588 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:12:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.562232  6588 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 0 FOLLOWER]: Advancing to term 1
06:12:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.572168  6588 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:35.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.574393  6588 leader_election.cc:290] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [CANDIDATE]: Term 1 election: Requested vote from peers 86dff63507b54089a717b2f1b12b249b (127.6.39.190:40895), 2a76e2b12b70451cb9836e7b10afef77 (127.6.39.189:43971)
06:12:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.575417  6457 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9145881f908a44b8b2a0a469ad677d6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a76e2b12b70451cb9836e7b10afef77"
06:12:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.575966  6457 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [term 0 FOLLOWER]: Advancing to term 1
06:12:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.583590  6457 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9145881f908a44b8b2a0a469ad677d6e in term 1.
06:12:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.584645  6518 leader_election.cc:304] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a76e2b12b70451cb9836e7b10afef77, 9145881f908a44b8b2a0a469ad677d6e; no voters: 
06:12:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.585584  6588 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 1 FOLLOWER]: Leader election won for term 1
06:12:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.587641  6588 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [term 1 LEADER]: Becoming Leader. State: Replica: 9145881f908a44b8b2a0a469ad677d6e, State: Running, Role: LEADER
06:12:35.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.588608  6588 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:35.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.606186  6610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9145881f908a44b8b2a0a469ad677d6e. Latest consensus state: current_term: 1 leader_uuid: "9145881f908a44b8b2a0a469ad677d6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:35.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.606931  6610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: This master's current role is: LEADER
06:12:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.610078  6613 catalog_manager.cc:1531] Loading table and tablet metadata into memory...
06:12:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.617486  6613 catalog_manager.cc:1540] Initializing Kudu cluster ID...
06:12:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.640909  6457 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [term 1 FOLLOWER]: Refusing update from remote peer 9145881f908a44b8b2a0a469ad677d6e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:12:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.643532  6588 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:12:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.661592  6606 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9145881f908a44b8b2a0a469ad677d6e. Latest consensus state: current_term: 1 leader_uuid: "9145881f908a44b8b2a0a469ad677d6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.662266  6606 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [sys.catalog]: This master's current role is: FOLLOWER
06:12:35.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.666584  6610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9145881f908a44b8b2a0a469ad677d6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.667258  6610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: This master's current role is: LEADER
06:12:35.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.671406  6613 catalog_manager.cc:1384] Generated new cluster ID: 0b6de3105d8540f2b65335c16366591e
06:12:35.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.671679  6613 catalog_manager.cc:1551] Initializing Kudu internal certificate authority...
06:12:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.675308  6606 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9145881f908a44b8b2a0a469ad677d6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.675959  6606 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77 [sys.catalog]: This master's current role is: FOLLOWER
06:12:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.676867  6402 master_runner.cc:187] Error getting master registration for 127.6.39.190:40895: OK, instance_id { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" instance_seqno: 1786515151181510 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:12:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.707403  6402 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
06:12:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:35.718647  6625 catalog_manager.cc:1614] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:12:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:35.718986  6625 catalog_manager.cc:884] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:12:35.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.735561  6613 catalog_manager.cc:1407] Generated new certificate authority record
06:12:35.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.738109  6613 catalog_manager.cc:1560] Loading token signing keys...
06:12:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.764649  6613 catalog_manager.cc:6112] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e: Generated new TSK 0
06:12:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.767218  6613 catalog_manager.cc:1570] Initializing in-progress tserver states...
06:12:35.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.832211  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:12:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:0
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:12:35.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.876542  6400 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } attempt: 7
06:12:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.929728  6400 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b: Bootstrap starting.
06:12:35.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.946859  6400 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b: Neither blocks nor log segments found. Creating new log.
06:12:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.952323  6400 log.cc:826] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b: Log is configured to *not* fsync() on all Append() calls
06:12:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:35.974156  6400 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b: No bootstrap required, opened a new log
06:12:36.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.027326  6400 raft_consensus.cc:359] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:36.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.028170  6400 raft_consensus.cc:385] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:12:36.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.028456  6400 raft_consensus.cc:740] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86dff63507b54089a717b2f1b12b249b, State: Initialized, Role: FOLLOWER
06:12:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.029407  6400 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } }
06:12:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.040138  6400 sys_catalog.cc:565] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [sys.catalog]: configured and running, proceeding with master startup.
06:12:36.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.040732  6363 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [term 0 FOLLOWER]: Advancing to term 1
06:12:36.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.045603  6365 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9145881f908a44b8b2a0a469ad677d6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86dff63507b54089a717b2f1b12b249b" is_pre_election: true
06:12:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.047106  6364 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9145881f908a44b8b2a0a469ad677d6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86dff63507b54089a717b2f1b12b249b"
06:12:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.051488  6363 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [term 1 FOLLOWER]: Refusing update from remote peer 9145881f908a44b8b2a0a469ad677d6e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:12:36.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.053301  6610 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [LEADER]: Connected to new peer: Peer: permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:12:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.056372  6633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.057130  6633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [sys.catalog]: This master's current role is: FOLLOWER
06:12:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.115417  6633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9145881f908a44b8b2a0a469ad677d6e. Latest consensus state: current_term: 1 leader_uuid: "9145881f908a44b8b2a0a469ad677d6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.116080  6633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [sys.catalog]: This master's current role is: FOLLOWER
06:12:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.117465  6643 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9145881f908a44b8b2a0a469ad677d6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.118552  6643 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b [sys.catalog]: This master's current role is: FOLLOWER
06:12:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.150439  6652 catalog_manager.cc:806] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
06:12:36.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:36.341341  6630 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:36.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:36.342325  6630 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:36.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:36.342954  6630 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:36.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:36.375042  6630 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:36.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:36.376065  6630 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.129
06:12:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.411784  6630 tablet_server_runner.cc:78] Tablet server non-default flags:
06:12:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:0
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:12:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:12:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.418358  6630 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:36.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.420836  6630 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:36.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:36.457779  6661 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:36.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.559315  6665 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9145881f908a44b8b2a0a469ad677d6e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dff63507b54089a717b2f1b12b249b" member_type: VOTER last_known_addr { host: "127.6.39.190" port: 40895 } } peers { permanent_uuid: "2a76e2b12b70451cb9836e7b10afef77" member_type: VOTER last_known_addr { host: "127.6.39.189" port: 43971 } } peers { permanent_uuid: "9145881f908a44b8b2a0a469ad677d6e" member_type: VOTER last_known_addr { host: "127.6.39.188" port: 40371 } } }
06:12:36.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.560791  6665 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9145881f908a44b8b2a0a469ad677d6e [sys.catalog]: This master's current role is: LEADER
06:12:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.725682  6625 catalog_manager.cc:1296] Loaded cluster ID: 0b6de3105d8540f2b65335c16366591e
06:12:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.735189  6625 catalog_manager.cc:1608] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: loading cluster ID for follower catalog manager: success
06:12:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.746634  6625 catalog_manager.cc:1630] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: acquiring CA information for follower catalog manager: success
06:12:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:36.762790  6625 catalog_manager.cc:1658] T 00000000000000000000000000000000 P 2a76e2b12b70451cb9836e7b10afef77: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:12:37.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:37.186800  6652 catalog_manager.cc:1296] Loaded cluster ID: 0b6de3105d8540f2b65335c16366591e
06:12:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:37.188416  6652 catalog_manager.cc:1608] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b: loading cluster ID for follower catalog manager: success
06:12:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:37.235136  6652 catalog_manager.cc:1630] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b: acquiring CA information for follower catalog manager: success
06:12:37.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:37.286724  6652 catalog_manager.cc:1658] T 00000000000000000000000000000000 P 86dff63507b54089a717b2f1b12b249b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:12:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:36.462039  6660 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:37.848008  6659 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 6630
06:12:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:38.322928  6630 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.867s	user 0.717s	sys 1.048s
06:12:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:38.324774  6630 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.869s	user 0.717s	sys 1.048s
06:12:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.340544  6630 server_base.cc:1061] running on GCE node
06:12:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:38.341925  6669 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.343636  6630 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:38.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.351239  6630 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.352932  6630 hybrid_clock.cc:648] HybridClock initialized: now 1786515158352887 us; error 43 us; skew 500 ppm
06:12:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.354264  6630 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.367848  6630 webserver.cc:533] Webserver started at http://127.6.39.129:45273/ using document root <none> and password file <none>
06:12:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.369071  6630 fs_manager.cc:362] Metadata directory not provided
06:12:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.369366  6630 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.369906  6630 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
06:12:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.376076  6630 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/instance:
06:12:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:12:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:38 on dist-test-slave-vq2q"
06:12:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.377559  6630 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal/instance:
06:12:38.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:12:38.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:38 on dist-test-slave-vq2q"
06:12:38.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.387468  6630 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.006s	sys 0.003s
06:12:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.394975  6674 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.396389  6630 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.004s	sys 0.001s
06:12:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.396795  6630 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:12:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:38 on dist-test-slave-vq2q"
06:12:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.397248  6630 fs_report.cc:389] FS layout report
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/data
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.472353  6630 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.474469  6630 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:38.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.474998  6630 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:38.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.479334  6630 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:12:38.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.485370  6630 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:12:38.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.485636  6630 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:12:38.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.485919  6630 ts_tablet_manager.cc:616] Registered 0 tablets
06:12:38.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.486127  6630 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:12:38.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.662925  6630 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.129:41443
06:12:38.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.663625  6811 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.129:41443 every 8 connection(s)
06:12:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.666327  6630 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:12:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.674829  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 6630
06:12:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.675266  6302 external_mini_cluster.cc:1449] Reading /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal/instance
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.691582  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:0
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:12:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.729965  6812 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:12:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.730468  6812 heartbeater.cc:461] Registering TS with master...
06:12:38.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.732390  6812 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:12:38.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.735795  6535 ts_manager.cc:194] Registered new tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.737332  6816 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:12:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.737633  6816 heartbeater.cc:461] Registering TS with master...
06:12:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.738502  6816 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:12:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.738476  6535 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.129:38453
06:12:38.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.741540  6354 ts_manager.cc:194] Registered new tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.743713  6813 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.744031  6813 heartbeater.cc:461] Registering TS with master...
06:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.744588  6813 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:12:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:38.747262  6447 ts_manager.cc:194] Registered new tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:39.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.051615  6819 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:39.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.052188  6819 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:39.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.052827  6819 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:39.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.085507  6819 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:39.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.086464  6819 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.130
06:12:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:39.123235  6819 tablet_server_runner.cc:78] Tablet server non-default flags:
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:0
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:12:39.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:39.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:39.124895  6819 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:39.127075  6819 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.146080  6836 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:39.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:39.743649  6812 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:12:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.155853  6838 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:39.153223  6834 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:40.432847  6837 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1281 milliseconds
06:12:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.432994  6819 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.434239  6819 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:40.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.436380  6819 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:40.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.437716  6819 hybrid_clock.cc:648] HybridClock initialized: now 1786515160437690 us; error 44 us; skew 500 ppm
06:12:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.438500  6819 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.444677  6819 webserver.cc:533] Webserver started at http://127.6.39.130:42227/ using document root <none> and password file <none>
06:12:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.445595  6819 fs_manager.cc:362] Metadata directory not provided
06:12:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.445806  6819 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.446185  6819 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
06:12:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.450512  6819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/instance:
06:12:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a785279bacd40deb619d9f334289da6"
06:12:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:40 on dist-test-slave-vq2q"
06:12:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.451570  6819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal/instance:
06:12:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a785279bacd40deb619d9f334289da6"
06:12:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:40 on dist-test-slave-vq2q"
06:12:40.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.458811  6819 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.003s
06:12:40.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.464540  6852 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.465751  6819 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.466045  6819 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a785279bacd40deb619d9f334289da6"
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:40 on dist-test-slave-vq2q"
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.466363  6819 fs_report.cc:389] FS layout report
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/data
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.519642  6819 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.521145  6819 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.521589  6819 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.524240  6819 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:12:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.528450  6819 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:12:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.528644  6819 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:12:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.528838  6819 ts_tablet_manager.cc:616] Registered 0 tablets
06:12:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.528967  6819 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:12:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.685398  6819 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.130:42467
06:12:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.685771  6989 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.130:42467 every 8 connection(s)
06:12:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.688301  6819 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:12:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.696964  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 6819
06:12:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.697805  6302 external_mini_cluster.cc:1449] Reading /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal/instance
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.713795  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:0
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:12:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:12:40.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.773403  6991 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:12:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.774008  6991 heartbeater.cc:461] Registering TS with master...
06:12:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.775257  6990 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:12:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.775638  6990 heartbeater.cc:461] Registering TS with master...
06:12:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.775744  6991 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:12:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.776736  6990 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:12:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.779389  6535 ts_manager.cc:194] Registered new tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:12:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.779458  6447 ts_manager.cc:194] Registered new tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:12:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.781250  6535 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.130:47191
06:12:40.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.786624  6993 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:12:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.787104  6993 heartbeater.cc:461] Registering TS with master...
06:12:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.788106  6993 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:12:40.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:40.791658  6354 ts_manager.cc:194] Registered new tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:12:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:41.083701  6997 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:41.084359  6997 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:41.085047  6997 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:41.117478  6997 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:41.118263  6997 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.131
06:12:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:41.154736  6997 tablet_server_runner.cc:78] Tablet server non-default flags:
06:12:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:41.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:12:41.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:0
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:41.156389  6997 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:41.158614  6997 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:41.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:41.172817  7010 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:41.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:41.786109  6990 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:12:42.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:41.173807  7012 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:42.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:42.292613  7013 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:12:42.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:42.294119  7014 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:42.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.294138  6997 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:42.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.297232  6997 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:42.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.299870  6997 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:42.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.301287  6997 hybrid_clock.cc:648] HybridClock initialized: now 1786515162301234 us; error 48 us; skew 500 ppm
06:12:42.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.302057  6997 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.308293  6997 webserver.cc:533] Webserver started at http://127.6.39.131:41543/ using document root <none> and password file <none>
06:12:42.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.309262  6997 fs_manager.cc:362] Metadata directory not provided
06:12:42.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.309536  6997 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:42.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.310045  6997 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
06:12:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.314218  6997 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/instance:
06:12:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:12:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:42 on dist-test-slave-vq2q"
06:12:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.315408  6997 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal/instance:
06:12:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:12:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:42 on dist-test-slave-vq2q"
06:12:42.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.322561  6997 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.000s	sys 0.008s
06:12:42.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.328114  7022 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.329259  6997 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
06:12:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.329537  6997 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:12:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:42 on dist-test-slave-vq2q"
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.329833  6997 fs_report.cc:389] FS layout report
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/data
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:42.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.384356  6997 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:42.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.385876  6997 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:42.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.386262  6997 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:42.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.388865  6997 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:12:42.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.392985  6997 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:12:42.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.393194  6997 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
06:12:42.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.393416  6997 ts_tablet_manager.cc:616] Registered 0 tablets
06:12:42.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.393551  6997 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
06:12:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.695658  6997 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.131:35835
06:12:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.696297  7162 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.131:35835 every 8 connection(s)
06:12:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.698968  6997 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:12:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.706295  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 6997
06:12:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.706751  6302 external_mini_cluster.cc:1449] Reading /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal/instance
06:12:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.751569  7163 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:12:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.752112  7163 heartbeater.cc:461] Registering TS with master...
06:12:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.754573  7163 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:12:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.759850  7165 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:12:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.760279  7165 heartbeater.cc:461] Registering TS with master...
06:12:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.760205  7164 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:12:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.760890  7164 heartbeater.cc:461] Registering TS with master...
06:12:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.762437  6535 ts_manager.cc:194] Registered new tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.764219  6535 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.131:42681
06:12:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.766057  7165 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:12:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.766631  7164 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:12:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.772001  6353 ts_manager.cc:194] Registered new tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:42.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.778023  6447 ts_manager.cc:194] Registered new tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:42.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:42.791290  6302 external_mini_cluster.cc:956] 3 TS(s) registered with all masters
06:12:42.811 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
06:12:42.812 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

06:12:42.823 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.6.39.190"
      port: 40895
    }
    bound_http_address {
      host: "127.6.39.190"
      port: 39455
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.6.39.189"
      port: 43971
    }
    bound_http_address {
      host: "127.6.39.189"
      port: 45929
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.6.39.188"
      port: 40371
    }
    bound_http_address {
      host: "127.6.39.188"
      port: 41011
    }
  }
}

06:12:42.826 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

06:12:42.829 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.6.39.129"
      port: 41443
    }
    bound_http_address {
      host: "127.6.39.129"
      port: 45273
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.6.39.130"
      port: 42467
    }
    bound_http_address {
      host: "127.6.39.130"
      port: 42227
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.6.39.131"
      port: 35835
    }
    bound_http_address {
      host: "127.6.39.131"
      port: 41543
    }
  }
}

06:12:42.829 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:12:43.032 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
06:12:43.081 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
06:12:43.273 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.39.189:43971(127.6.39.189:43971)] connecting to peer
06:12:43.323 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.39.190:40895(127.6.39.190:40895)] connecting to peer
06:12:43.334 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.39.188:40371(127.6.39.188:40371)] connecting to peer
06:12:43.340 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.6.39.190:40895(127.6.39.190:40895)] Successfully connected to peer
06:12:43.342 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.39.189:43971(127.6.39.189:43971)] Successfully connected to peer
06:12:43.348 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.6.39.188:40371(127.6.39.188:40371)] Successfully connected to peer
06:12:43.465 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.6.39.189
06:12:43.465 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.6.39.188
06:12:43.466 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.6.39.190
06:12:43.684 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x9b34f1f9, L:/127.0.0.1:54858 - R:/127.6.39.188:40371] using SASL/PLAIN
06:12:43.684 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x9646f7dd, L:/127.0.0.1:38056 - R:/127.6.39.190:40895] using SASL/PLAIN
06:12:43.684 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x9f6baef7, L:/127.0.0.1:60460 - R:/127.6.39.189:43971] using SASL/PLAIN
06:12:43.723 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:12:43.724 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=489), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.39.189:43971, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.39.190:40895, [94ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.39.188:40371, [463ms] Sub RPC ConnectToMaster: received response from server master-127.6.39.190:40895: OK, [463ms] Sub RPC ConnectToMaster: received response from server master-127.6.39.189:43971: OK, [464ms] Sub RPC ConnectToMaster: received response from server master-127.6.39.188:40371: OK, deferred=null) after lookup
06:12:43.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.776098  7163 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:12:43.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.777220  6533 catalog_manager.cc:2303] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54858:
06:12:43.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217"
06:12:43.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:12:43.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:12:43.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
06:12:43.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
06:12:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
06:12:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:12:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:43.785298  6533 catalog_manager.cc:7119] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:12:43.889 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 06d5cb681c624bee8859557e62435e84
06:12:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.910240  6731 tablet_service.cc:1511] Processing CreateTablet for tablet b27f111dffc74b94ba5a7a2a6d8af055 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217 [id=06d5cb681c624bee8859557e62435e84]), partition=RANGE (key) PARTITION UNBOUNDED
06:12:43.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.912189  6731 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b27f111dffc74b94ba5a7a2a6d8af055. 1 dirs total, 0 dirs full, 0 dirs failed
06:12:43.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.912038  6910 tablet_service.cc:1511] Processing CreateTablet for tablet b27f111dffc74b94ba5a7a2a6d8af055 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217 [id=06d5cb681c624bee8859557e62435e84]), partition=RANGE (key) PARTITION UNBOUNDED
06:12:43.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.914219  6910 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b27f111dffc74b94ba5a7a2a6d8af055. 1 dirs total, 0 dirs full, 0 dirs failed
06:12:43.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.918125  7085 tablet_service.cc:1511] Processing CreateTablet for tablet b27f111dffc74b94ba5a7a2a6d8af055 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217 [id=06d5cb681c624bee8859557e62435e84]), partition=RANGE (key) PARTITION UNBOUNDED
06:12:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.920033  7085 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b27f111dffc74b94ba5a7a2a6d8af055. 1 dirs total, 0 dirs full, 0 dirs failed
06:12:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.952605  7209 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap starting.
06:12:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.960976  7210 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap starting.
06:12:43.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.965265  7209 tablet_bootstrap.cc:654] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Neither blocks nor log segments found. Creating new log.
06:12:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.968587  7210 tablet_bootstrap.cc:654] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Neither blocks nor log segments found. Creating new log.
06:12:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.968776  7209 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Log is configured to *not* fsync() on all Append() calls
06:12:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.971350  7210 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Log is configured to *not* fsync() on all Append() calls
06:12:43.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.980520  7209 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: No bootstrap required, opened a new log
06:12:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.981344  7209 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent bootstrapping tablet: real 0.029s	user 0.021s	sys 0.000s
06:12:43.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.990559  7210 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: No bootstrap required, opened a new log
06:12:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.991420  7210 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent bootstrapping tablet: real 0.032s	user 0.012s	sys 0.010s
06:12:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:43.991677  7213 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap starting.
06:12:44.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.005988  7213 tablet_bootstrap.cc:654] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Neither blocks nor log segments found. Creating new log.
06:12:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.010052  7213 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Log is configured to *not* fsync() on all Append() calls
06:12:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.016796  7209 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.018205  7209 raft_consensus.cc:385] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:12:44.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.018767  7209 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Initialized, Role: FOLLOWER
06:12:44.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.020509  7209 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.028286  7210 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.029855  7210 raft_consensus.cc:385] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:12:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.030545  7210 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7a785279bacd40deb619d9f334289da6, State: Initialized, Role: FOLLOWER
06:12:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.031947  7209 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent starting tablet: real 0.050s	user 0.039s	sys 0.004s
06:12:44.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.032135  7210 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.034502  7213 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: No bootstrap required, opened a new log
06:12:44.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.035084  7213 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent bootstrapping tablet: real 0.044s	user 0.012s	sys 0.015s
06:12:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.062427  7210 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent starting tablet: real 0.070s	user 0.040s	sys 0.016s
06:12:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.085985  7213 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.087263  7213 raft_consensus.cc:385] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:12:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.087639  7213 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Initialized, Role: FOLLOWER
06:12:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.089059  7213 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.095444  7213 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent starting tablet: real 0.056s	user 0.022s	sys 0.021s
06:12:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:44.232599  7172 tablet.cc:2406] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:12:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:44.233493  6817 tablet.cc:2406] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:12:44.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:44.244648  6995 tablet.cc:2406] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:12:44.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.302968  7216 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:12:44.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.303709  7216 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.310261  7216 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.326181  6756 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4" is_pre_election: true
06:12:44.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.327167  6756 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 0.
06:12:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.327389  6938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6" is_pre_election: true
06:12:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.328315  6938 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 0.
06:12:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.328789  7023 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, e93dadc87c1d4f888622d646f01f4da6; no voters: 
06:12:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.330109  7216 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:12:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.330403  7216 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:12:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.330729  7216 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 0 FOLLOWER]: Advancing to term 1
06:12:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.338785  7216 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.341537  7216 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 1 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.342228  6938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6"
06:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.342690  6938 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 0 FOLLOWER]: Advancing to term 1
06:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.342607  6756 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.343096  6756 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 0 FOLLOWER]: Advancing to term 1
06:12:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.347114  6938 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 1.
06:12:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.347433  6756 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 1.
06:12:44.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.348147  7026 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6; no voters: 
06:12:44.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.348874  7216 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Leader election won for term 1
06:12:44.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.351644  7216 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 LEADER]: Becoming Leader. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Running, Role: LEADER
06:12:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.352707  7216 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.368157  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 reported cstate change: term changed from 0 to 1, leader changed from <none> to e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131). New cstate: current_term: 1 leader_uuid: "e93dadc87c1d4f888622d646f01f4da6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: UNKNOWN } } }
06:12:44.436 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 06d5cb681c624bee8859557e62435e84
06:12:44.455 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:44.460 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:44.462 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [9ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=null) after lookup
06:12:44.471 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:44.474 [DEBUG - kudu-nio-3] (Connection.java:757) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] Successfully connected to peer
06:12:44.562 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x42b7ba9f, L:/127.0.0.1:44008 - R:/127.6.39.131:35835] using TOKEN/null
06:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.579068  6756 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 1 FOLLOWER]: Refusing update from remote peer e93dadc87c1d4f888622d646f01f4da6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.579169  6938 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 1 FOLLOWER]: Refusing update from remote peer e93dadc87c1d4f888622d646f01f4da6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:12:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.580834  7216 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:12:44.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.581594  7222 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:12:44.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.612207  7237 mvcc.cc:204] Tried to move back new op lower bound from 7317566114097045504 to 7317566113197887488. Current Snapshot: MvccSnapshot[applied={T|T < 7317566114097045504}]
06:12:44.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.629009  7244 mvcc.cc:204] Tried to move back new op lower bound from 7317566114097045504 to 7317566113197887488. Current Snapshot: MvccSnapshot[applied={T|T < 7317566114097045504}]
06:12:44.841 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:12:44.843 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.131:35835
06:12:44.844 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:12:44.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:44.845309  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 6997
06:12:44.886 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:12:44.888 [DEBUG - kudu-nio-3] (Connection.java:239) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] handling channelInactive
06:12:44.888 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state READY due to: connection closed
06:12:44.889 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: connection closed
06:12:44.900 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:44.900 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:44.901 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1492229859(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@fd23130 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2fd56f9b -> wakeup thread Time-limited test)) after lookup
06:12:44.902 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:44.905 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.905 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.905 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.920 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:44.921 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:44.921 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [18ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [18ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1492229859(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@fd23130 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2fd56f9b -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882))) after lookup
06:12:44.922 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:44.924 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.924 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.924 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.960 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:44.960 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:44.961 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [18ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [18ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1492229859(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@fd23130 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2fd56f9b -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115))) after lookup
06:12:44.961 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:44.963 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.963 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:44.963 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:45.060 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:45.060 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:45.061 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=173), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [18ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [18ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [74ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [76ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [76ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [168ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [173ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1492229859(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@fd23130 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115) -> (continuation of Deferred@275025069 after retry RPC@1685256112), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2fd56f9b -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115) -> (continuation of Deferred@275025069 after retry RPC@1685256112))) after lookup
06:12:45.062 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:45.064 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:45.064 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:45.064 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:45.560 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:45.560 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:45.561 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=674), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [18ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [18ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [74ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [76ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [76ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [168ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [173ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [174ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [177ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [177ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [668ms] refreshing cache from master, [668ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [673ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1492229859(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@fd23130 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115) -> (continuation of Deferred@275025069 after retry RPC@1685256112) -> (continuation of Deferred@825944466 after retry RPC@1665600625), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2fd56f9b -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115) -> (continuation of Deferred@275025069 after retry RPC@1685256112) -> (continuation of Deferred@825944466 after retry RPC@1665600625))) after lookup
06:12:45.562 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:45.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:45.563 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:45.563 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.227553  7258 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:12:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.227982  7258 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:46.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.231024  7258 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:46.235882  6679 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:12:46.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:46.242055  6679 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:12:46.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.242878  6938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6" is_pre_election: true
06:12:46.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.243432  6938 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 1.
06:12:46.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.244439  6679 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:12:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.245162  7258 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 1 FOLLOWER]: Leader pre-election won for term 2
06:12:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.245477  7258 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 1 FOLLOWER]: Starting leader election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:12:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.246169  7258 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 1 FOLLOWER]: Advancing to term 2
06:12:46.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.250623  7258 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.252223  7258 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 2 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.253062  6938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6"
06:12:46.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.253633  6938 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 1 FOLLOWER]: Advancing to term 2
06:12:46.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:46.256271  6679 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:12:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.258998  6938 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 2.
06:12:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.259904  6679 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:12:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.260502  7258 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 2 FOLLOWER]: Leader election won for term 2
06:12:46.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.262269  7258 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 2 LEADER]: Becoming Leader. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Running, Role: LEADER
06:12:46.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.263347  7258 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.271903  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 reported cstate change: term changed from 1 to 2, leader changed from e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131) to 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129). New cstate: current_term: 2 leader_uuid: "08e4006c4cde42189b7529a3c7455cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: HEALTHY } } }
06:12:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.764550  6938 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 2 FOLLOWER]: Refusing update from remote peer 08e4006c4cde42189b7529a3c7455cc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
06:12:46.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.766047  7258 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
06:12:46.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:46.795578  6679 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 -> Peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Couldn't send request to peer e93dadc87c1d4f888622d646f01f4da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:12:46.819 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:46.819 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:46.820 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1932), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [2ms] delaying RPC due to: Network error: connection closed, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [18ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [18ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [74ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [76ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [76ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [168ms] refreshing cache from master, [169ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [173ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [174ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [177ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [177ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [668ms] refreshing cache from master, [668ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [673ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [675ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [676ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [677ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [1928ms] refreshing cache from master, [1928ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1932ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1492229859(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@fd23130 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115) -> (continuation of Deferred@275025069 after retry RPC@1685256112) -> (continuation of Deferred@825944466 after retry RPC@1665600625) -> (continuation of Deferred@514161641 after retry RPC@1171970248), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2fd56f9b -> wakeup thread Time-limited test -> (continuation of Deferred@2120934633 after retry RPC@1252071882) -> (continuation of Deferred@279731778 after retry RPC@1903004115) -> (continuation of Deferred@275025069 after retry RPC@1685256112) -> (continuation of Deferred@825944466 after retry RPC@1665600625) -> (continuation of Deferred@514161641 after retry RPC@1171970248))) after lookup
06:12:46.821 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:46.822 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] Successfully connected to peer
06:12:46.849 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x3fd5422b, L:/127.0.0.1:45760 - R:/127.6.39.129:41443] using TOKEN/null
06:12:46.897 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.131:35835
06:12:46.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:46.900938  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:12:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.198777  7294 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.199329  7294 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:47.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.199823  7294 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:47.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.231297  7294 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:47.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.232117  7294 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.131
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:47.269753  7294 tablet_server_runner.cc:78] Tablet server non-default flags:
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:12:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:47.271162  7294 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:47.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:47.273370  7294 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.288678  7301 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.293337  7304 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:47.291437  7302 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:48.508178  7303 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1214 milliseconds
06:12:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.508301  7294 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.509636  7294 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:48.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.512146  7294 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.513540  7294 hybrid_clock.cc:648] HybridClock initialized: now 1786515168513493 us; error 58 us; skew 500 ppm
06:12:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.514380  7294 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:48.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.520232  7294 webserver.cc:533] Webserver started at http://127.6.39.131:41543/ using document root <none> and password file <none>
06:12:48.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.521149  7294 fs_manager.cc:362] Metadata directory not provided
06:12:48.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.521436  7294 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:48.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.529436  7294 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.005s
06:12:48.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.533960  7318 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:48.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.535029  7294 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
06:12:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.535365  7294 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:12:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:42 on dist-test-slave-vq2q"
06:12:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.537317  7294 fs_report.cc:389] FS layout report
06:12:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/data
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:48.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.610750  7294 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.612385  7294 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.612903  7294 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:48.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.615644  7294 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:12:48.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.621393  7325 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:12:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.629901  7294 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:12:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.630160  7294 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
06:12:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.630482  7294 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:12:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.635274  7294 ts_tablet_manager.cc:616] Registered 1 tablets
06:12:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.635466  7294 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
06:12:48.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.635848  7325 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap starting.
06:12:48.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.698405  7325 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Log is configured to *not* fsync() on all Append() calls
06:12:48.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.729784  7325 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:12:48.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.730787  7325 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap complete.
06:12:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.732129  7325 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent bootstrapping tablet: real 0.097s	user 0.064s	sys 0.027s
06:12:48.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.755862  7325 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:48.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.759462  7325 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Initialized, Role: FOLLOWER
06:12:48.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.760663  7325 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.766433  7325 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent starting tablet: real 0.034s	user 0.019s	sys 0.014s
06:12:48.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.818349  7294 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.131:35835
06:12:48.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.818699  7463 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.131:35835 every 8 connection(s)
06:12:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.821031  7294 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:12:48.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.826529  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 7294
06:12:48.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:12:48.849 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:12:48.850 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.129:41443
06:12:48.850 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:12:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.850984  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 6630
06:12:48.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.879976  7466 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:12:48.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.880478  7466 heartbeater.cc:461] Registering TS with master...
06:12:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.881727  7466 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:12:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.883754  7464 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:12:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.884099  7464 heartbeater.cc:461] Registering TS with master...
06:12:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.884936  7464 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:12:48.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.890502  7467 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:12:48.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.890424  6535 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:48.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.890852  7467 heartbeater.cc:461] Registering TS with master...
06:12:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.891732  7467 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:12:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.892017  6446 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:48.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.893477  6535 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.131:52213
06:12:48.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:48.895087  6353 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:48.902 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] handling channelInactive
06:12:48.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state READY due to: connection closed
06:12:48.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:12:48.906 [DEBUG - Time-limited test] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:48.907 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.907 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.907 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.919 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:48.919 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:48.919 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test)) after lookup
06:12:48.920 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:48.921 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.921 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.921 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.940 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:48.940 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:48.940 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [16ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258))) after lookup
06:12:48.941 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:48.942 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.942 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.942 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.959 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:48.960 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:48.960 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [16ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370))) after lookup
06:12:48.961 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:48.962 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.962 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:48.962 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:49.060 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:49.060 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=154), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [16ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [56ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [57ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [57ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905))) after lookup
06:12:49.061 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:49.062 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.062 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.062 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.180 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:49.180 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:49.181 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=275), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [16ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [56ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [57ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [57ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [156ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [157ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [157ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [275ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585))) after lookup
06:12:49.182 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:49.183 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.183 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.184 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.600 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:49.600 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:49.603 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=697), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [16ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [56ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [57ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [57ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [156ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [157ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [157ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [275ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [276ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [278ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [279ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [690ms] refreshing cache from master, [691ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [694ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585) -> (continuation of Deferred@777325329 after retry RPC@2130696468), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585) -> (continuation of Deferred@777325329 after retry RPC@2130696468))) after lookup
06:12:49.604 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:49.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.605 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:49.898790  7464 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:12:50.080 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:50.081 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:50.081 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1175), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [16ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [56ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [57ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [57ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [156ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [157ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [157ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [275ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [276ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [278ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [279ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [690ms] refreshing cache from master, [691ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [694ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [698ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [700ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [700ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1170ms] refreshing cache from master, [1170ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1175ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585) -> (continuation of Deferred@777325329 after retry RPC@2130696468) -> (continuation of Deferred@327399198 after retry RPC@322239235), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585) -> (continuation of Deferred@777325329 after retry RPC@2130696468) -> (continuation of Deferred@327399198 after retry RPC@322239235))) after lookup
06:12:50.082 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:12:50.083 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:50.083 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:50.084 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:12:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.091194  7481 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:12:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.091651  7481 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.094182  7481 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:50.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:50.098042  7319 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:12:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:50.102231  7319 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:12:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.106055  6938 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6" is_pre_election: true
06:12:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.107700  7322 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6
06:12:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.108379  7481 raft_consensus.cc:2749] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
06:12:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.186208  7487 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 08e4006c4cde42189b7529a3c7455cc4)
06:12:50.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.187136  7487 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:50.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.190003  7487 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:50.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:50.196056  6853 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:12:50.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:50.200579  6853 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:12:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.203614  7406 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "7a785279bacd40deb619d9f334289da6" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "e93dadc87c1d4f888622d646f01f4da6" is_pre_election: true
06:12:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.204231  7406 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7a785279bacd40deb619d9f334289da6 in term 1.
06:12:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.205596  6856 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4
06:12:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.206278  7487 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 2 FOLLOWER]: Leader pre-election won for term 3
06:12:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.206598  7487 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 08e4006c4cde42189b7529a3c7455cc4)
06:12:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.206817  7487 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 2 FOLLOWER]: Advancing to term 3
06:12:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.211402  7487 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.212865  7487 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 3 election: Requested vote from peers e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.213606  7406 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "7a785279bacd40deb619d9f334289da6" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:12:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.214133  7406 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 1 FOLLOWER]: Advancing to term 3
06:12:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:50.217216  6853 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:12:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.223048  7406 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7a785279bacd40deb619d9f334289da6 in term 3.
06:12:50.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.224139  6856 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4
06:12:50.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.224799  7487 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 3 FOLLOWER]: Leader election won for term 3
06:12:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.226761  7487 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 3 LEADER]: Becoming Leader. State: Replica: 7a785279bacd40deb619d9f334289da6, State: Running, Role: LEADER
06:12:50.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.227902  7487 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.238762  6535 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 reported cstate change: term changed from 2 to 3, leader changed from 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129) to 7a785279bacd40deb619d9f334289da6 (127.6.39.130). New cstate: current_term: 3 leader_uuid: "7a785279bacd40deb619d9f334289da6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: UNKNOWN } } }
06:12:50.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:50.670985  6853 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 -> Peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Couldn't send request to peer 08e4006c4cde42189b7529a3c7455cc4. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:12:50.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.834808  7406 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 3 FOLLOWER]: Refusing update from remote peer 7a785279bacd40deb619d9f334289da6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
06:12:50.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:50.836330  7487 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
06:12:52.380 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:52.380 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:52.381 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=16, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=3475), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [2ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [13ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [14ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [16ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [56ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [57ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [57ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [156ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [157ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [157ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [270ms] refreshing cache from master, [270ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [275ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [276ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [278ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [279ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [690ms] refreshing cache from master, [691ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [694ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [698ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [700ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [700ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1170ms] refreshing cache from master, [1170ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1175ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [1177ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1178ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1179ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3470ms] refreshing cache from master, [3470ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [3474ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@811788690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@446b37de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585) -> (continuation of Deferred@777325329 after retry RPC@2130696468) -> (continuation of Deferred@327399198 after retry RPC@322239235) -> (continuation of Deferred@718481754 after retry RPC@504046560), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7f6c374a -> wakeup thread Time-limited test -> (continuation of Deferred@287177492 after retry RPC@257247258) -> (continuation of Deferred@471731227 after retry RPC@959353370) -> (continuation of Deferred@1626614327 after retry RPC@763873905) -> (continuation of Deferred@2096248292 after retry RPC@1865792585) -> (continuation of Deferred@777325329 after retry RPC@2130696468) -> (continuation of Deferred@327399198 after retry RPC@322239235) -> (continuation of Deferred@718481754 after retry RPC@504046560))) after lookup
06:12:52.382 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:12:52.383 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] Successfully connected to peer
06:12:52.406 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xe061337b, L:/127.0.0.1:35592 - R:/127.6.39.130:42467] using TOKEN/null
06:12:52.454 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.129:41443
06:12:52.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:52.456604  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:41443
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:12:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45273
06:12:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:12:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:12:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.786062  7528 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.786590  7528 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:52.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.787065  7528 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.818374  7528 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.819180  7528 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.129
06:12:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:52.855468  7528 tablet_server_runner.cc:78] Tablet server non-default flags:
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:41443
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45273
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:52.856815  7528 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:52.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:52.858693  7528 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:52.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.871556  7536 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:53.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:53.090251  6853 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 -> Peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Couldn't send request to peer 08e4006c4cde42189b7529a3c7455cc4. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:12:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.874821  7539 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:52.878261  7537 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:54.001920  7538 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:12:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.002120  7528 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.005599  7528 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.007812  7528 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.009224  7528 hybrid_clock.cc:648] HybridClock initialized: now 1786515174009160 us; error 50 us; skew 500 ppm
06:12:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.010152  7528 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.016716  7528 webserver.cc:533] Webserver started at http://127.6.39.129:45273/ using document root <none> and password file <none>
06:12:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.017710  7528 fs_manager.cc:362] Metadata directory not provided
06:12:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.017977  7528 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:54.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.026394  7528 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.004s
06:12:54.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.031051  7552 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:54.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.032193  7528 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
06:12:54.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.032546  7528 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:54.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:12:54.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:38 on dist-test-slave-vq2q"
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.034516  7528 fs_report.cc:389] FS layout report
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/data
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:54.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.085527  7528 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.087158  7528 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.087594  7528 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:54.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.090152  7528 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:12:54.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.095623  7563 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:12:54.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.103876  7528 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:12:54.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.104092  7528 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
06:12:54.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.104321  7528 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:12:54.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.108732  7528 ts_tablet_manager.cc:616] Registered 1 tablets
06:12:54.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.108903  7528 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.001s	sys 0.000s
06:12:54.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.109393  7563 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap starting.
06:12:54.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.168416  7563 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Log is configured to *not* fsync() on all Append() calls
06:12:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.208074  7563 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:12:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.209240  7563 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap complete.
06:12:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.210870  7563 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent bootstrapping tablet: real 0.102s	user 0.064s	sys 0.035s
06:12:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.235258  7563 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.238663  7563 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Initialized, Role: FOLLOWER
06:12:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.240108  7563 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:54.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.247076  7563 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent starting tablet: real 0.036s	user 0.032s	sys 0.004s
06:12:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.290153  7528 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.129:41443
06:12:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.290419  7694 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.129:41443 every 8 connection(s)
06:12:54.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.292737  7528 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:12:54.296 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:12:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.295077  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 7528
06:12:54.324 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:12:54.325 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.130:42467
06:12:54.325 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:12:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.325909  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 6819
06:12:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.356778  7695 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:12:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.357357  7695 heartbeater.cc:461] Registering TS with master...
06:12:54.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.358772  7695 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:12:54.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.363195  6533 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:54.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.366978  7696 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:12:54.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.367316  7696 heartbeater.cc:461] Registering TS with master...
06:12:54.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.368244  7696 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:12:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.369474  6533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.129:48759
06:12:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.369822  7697 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:12:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.370187  7697 heartbeater.cc:461] Registering TS with master...
06:12:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.371134  7697 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:12:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.371824  6446 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:54.374583  6353 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:54.387 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] handling channelInactive
06:12:54.388 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state READY due to: connection closed
06:12:54.389 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:12:54.390 [DEBUG - Time-limited test] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:12:54.391 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.391 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.392 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.399 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:54.399 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:54.400 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [9ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@946582169(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@21a106d6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f116e7e -> wakeup thread Time-limited test)) after lookup
06:12:54.400 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:12:54.401 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.401 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.402 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.419 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:54.420 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:54.420 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [9ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [10ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [11ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [12ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [29ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@946582169(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@21a106d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f116e7e -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403))) after lookup
06:12:54.420 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:12:54.421 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.422 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.422 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.460 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:54.461 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:54.461 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [9ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [10ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [11ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [12ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [29ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [30ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [32ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [70ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@946582169(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@21a106d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f116e7e -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741))) after lookup
06:12:54.462 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:12:54.463 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.463 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.463 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.520 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:54.520 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:54.521 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=131), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [9ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [10ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [11ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [12ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [29ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [30ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [32ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [70ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [71ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [73ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [73ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [125ms] refreshing cache from master, [126ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [130ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@946582169(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@21a106d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741) -> (continuation of Deferred@1130395544 after retry RPC@167469311), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f116e7e -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741) -> (continuation of Deferred@1130395544 after retry RPC@167469311))) after lookup
06:12:54.521 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:12:54.522 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.523 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:54.523 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:55.039 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:55.040 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:12:55.040 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=650), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [9ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [10ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [11ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [12ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [29ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [30ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [32ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [70ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [71ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [73ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [73ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [125ms] refreshing cache from master, [126ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [130ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [131ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [133ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [133ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [645ms] refreshing cache from master, [645ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [649ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@946582169(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@21a106d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741) -> (continuation of Deferred@1130395544 after retry RPC@167469311) -> (continuation of Deferred@1370654872 after retry RPC@52225737), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f116e7e -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741) -> (continuation of Deferred@1130395544 after retry RPC@167469311) -> (continuation of Deferred@1370654872 after retry RPC@52225737))) after lookup
06:12:55.041 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:12:55.042 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:55.042 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:55.042 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:12:55.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.297726  7705 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 7a785279bacd40deb619d9f334289da6)
06:12:55.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.298148  7705 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:55.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.299582  7705 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:55.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:55.301928  7322 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:12:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:55.306535  7322 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:12:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.313853  7642 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4" is_pre_election: true
06:12:55.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.314720  7642 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 2.
06:12:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.316159  7319 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, e93dadc87c1d4f888622d646f01f4da6; no voters: 7a785279bacd40deb619d9f334289da6
06:12:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.316884  7705 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 3 FOLLOWER]: Leader pre-election won for term 4
06:12:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.317158  7705 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 7a785279bacd40deb619d9f334289da6)
06:12:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.317490  7705 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 3 FOLLOWER]: Advancing to term 4
06:12:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.321952  7705 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.323572  7705 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 4 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:12:55.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.324677  7642 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:12:55.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.325160  7642 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 2 FOLLOWER]: Advancing to term 4
06:12:55.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:55.327032  7322 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:12:55.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.331961  7642 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 4.
06:12:55.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.332880  7319 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, e93dadc87c1d4f888622d646f01f4da6; no voters: 7a785279bacd40deb619d9f334289da6
06:12:55.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.333582  7705 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 4 FOLLOWER]: Leader election won for term 4
06:12:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.334620  7705 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 4 LEADER]: Becoming Leader. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Running, Role: LEADER
06:12:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.335525  7705 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.344694  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 reported cstate change: term changed from 3 to 4, leader changed from 7a785279bacd40deb619d9f334289da6 (127.6.39.130) to e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131). New cstate: current_term: 4 leader_uuid: "e93dadc87c1d4f888622d646f01f4da6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: UNKNOWN } } }
06:12:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.374543  7695 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:12:55.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:55.726395  7322 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 -> Peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Couldn't send request to peer 7a785279bacd40deb619d9f334289da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:12:55.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.765514  7642 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 4 FOLLOWER]: Refusing update from remote peer e93dadc87c1d4f888622d646f01f4da6: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
06:12:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:55.766933  7705 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
06:12:56.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:56.060 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:56.060 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1670), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [9ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [10ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [11ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [12ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [29ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [30ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [32ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [70ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [71ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [73ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [73ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [125ms] refreshing cache from master, [126ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [130ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [131ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [133ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [133ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [645ms] refreshing cache from master, [645ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [649ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [650ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [652ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [652ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1665ms] refreshing cache from master, [1665ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1669ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@946582169(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@21a106d6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741) -> (continuation of Deferred@1130395544 after retry RPC@167469311) -> (continuation of Deferred@1370654872 after retry RPC@52225737) -> (continuation of Deferred@11336584 after retry RPC@1684172794), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@f116e7e -> wakeup thread Time-limited test -> (continuation of Deferred@1476373940 after retry RPC@1402559403) -> (continuation of Deferred@1220421882 after retry RPC@1365468741) -> (continuation of Deferred@1130395544 after retry RPC@167469311) -> (continuation of Deferred@1370654872 after retry RPC@52225737) -> (continuation of Deferred@11336584 after retry RPC@1684172794))) after lookup
06:12:56.061 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:56.062 [DEBUG - kudu-nio-1] (Connection.java:757) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] Successfully connected to peer
06:12:56.085 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xb288a7f9, L:/127.0.0.1:42698 - R:/127.6.39.131:35835] using TOKEN/null
06:12:56.128 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.130:42467
06:12:56.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:56.130012  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:12:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:42467
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42227
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:12:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.463734  7739 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:12:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.464310  7739 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:12:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.464916  7739 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:12:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.499274  7739 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:12:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.500298  7739 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.130
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:56.536957  7739 tablet_server_runner.cc:78] Tablet server non-default flags:
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:42467
06:12:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42227
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:12:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:56.538748  7739 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:12:56.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:56.540529  7739 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:12:56.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.554216  7747 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:57.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.555518  7748 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:57.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:56.559346  7750 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:12:57.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:57.760911  7749 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1201 milliseconds
06:12:57.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.761088  7739 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:12:57.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.762583  7739 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:12:57.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.765129  7739 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:12:57.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.766618  7739 hybrid_clock.cc:648] HybridClock initialized: now 1786515177766580 us; error 41 us; skew 500 ppm
06:12:57.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.767669  7739 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:12:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.775347  7739 webserver.cc:533] Webserver started at http://127.6.39.130:42227/ using document root <none> and password file <none>
06:12:57.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.776659  7739 fs_manager.cc:362] Metadata directory not provided
06:12:57.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.776995  7739 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:12:57.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.787618  7739 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.005s	sys 0.004s
06:12:57.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.792805  7767 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:12:57.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.794095  7739 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
06:12:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.794433  7739 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a785279bacd40deb619d9f334289da6"
06:12:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:40 on dist-test-slave-vq2q"
06:12:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.796394  7739 fs_report.cc:389] FS layout report
06:12:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/data
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:12:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:12:57.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.847681  7739 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:12:57.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.849318  7739 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:12:57.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.849800  7739 kserver.cc:163] Server-wide thread pool size limit: 3276
06:12:57.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.852514  7739 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:12:57.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.858695  7779 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:12:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.867378  7739 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:12:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.867602  7739 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
06:12:57.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.867916  7739 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:12:57.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.873459  7739 ts_tablet_manager.cc:616] Registered 1 tablets
06:12:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.873703  7739 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.003s	sys 0.003s
06:12:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.874554  7779 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap starting.
06:12:57.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.950043  7779 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Log is configured to *not* fsync() on all Append() calls
06:12:57.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.996353  7779 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:12:57.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.997318  7779 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap complete.
06:12:57.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:57.998720  7779 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent bootstrapping tablet: real 0.125s	user 0.080s	sys 0.039s
06:12:58.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.017830  7779 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:58.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.020490  7779 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7a785279bacd40deb619d9f334289da6, State: Initialized, Role: FOLLOWER
06:12:58.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.021621  7779 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:58.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.026919  7779 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent starting tablet: real 0.028s	user 0.024s	sys 0.004s
06:12:58.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.062301  7739 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.130:42467
06:12:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.062698  7912 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.130:42467 every 8 connection(s)
06:12:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.065384  7739 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:12:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.074813  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 7739
06:12:58.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:12:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.101264  7913 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:12:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.101713  7913 heartbeater.cc:461] Registering TS with master...
06:12:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.103001  7913 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:12:58.106 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:12:58.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.110780  6533 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:12:58.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.111464  7914 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:12:58.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.111824  7914 heartbeater.cc:461] Registering TS with master...
06:12:58.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.112735  7914 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:12:58.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.113672  6533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.130:57703
06:12:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.116813  6446 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:12:58.121 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.131:35835
06:12:58.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:12:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.122041  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 7294
06:12:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.129865  7915 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:12:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.130271  7915 heartbeater.cc:461] Registering TS with master...
06:12:58.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.130990  7915 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:12:58.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:58.134778  6354 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:12:58.166 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:12:58.166 [DEBUG - kudu-nio-1] (Connection.java:239) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] handling channelInactive
06:12:58.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state READY due to: connection closed
06:12:58.167 [DEBUG - Time-limited test] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:58.168 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.168 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.169 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.178 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:58.179 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:58.179 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@552497245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@701c018c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79590be4 -> wakeup thread Time-limited test)) after lookup
06:12:58.180 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:58.181 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.181 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.181 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.199 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:58.199 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:58.200 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [13ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [15ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [15ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@552497245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@701c018c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79590be4 -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893))) after lookup
06:12:58.200 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:58.201 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.202 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.202 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.239 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:58.239 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:58.239 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [13ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [15ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [15ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [34ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [36ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [36ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@552497245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@701c018c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79590be4 -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222))) after lookup
06:12:58.240 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:58.241 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.242 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.242 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.379 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:58.379 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:58.380 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=213), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [13ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [15ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [15ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [34ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [36ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [36ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [74ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [76ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [76ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [213ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@552497245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@701c018c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79590be4 -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774))) after lookup
06:12:58.381 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:58.382 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.382 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.382 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.800 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:58.800 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:58.800 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=634), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [13ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [15ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [15ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [34ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [36ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [36ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [74ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [76ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [76ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [213ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [214ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [216ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [216ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [629ms] refreshing cache from master, [630ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [633ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@552497245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@701c018c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774) -> (continuation of Deferred@832393612 after retry RPC@1660492861), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79590be4 -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774) -> (continuation of Deferred@832393612 after retry RPC@1660492861))) after lookup
06:12:58.801 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:58.802 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.802 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:58.802 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:59.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:12:59.060 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:12:59.060 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=893), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [13ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [15ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [15ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [34ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [36ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [36ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [74ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [76ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [76ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [213ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [214ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [216ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [216ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [629ms] refreshing cache from master, [630ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [633ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [635ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [636ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [637ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [889ms] refreshing cache from master, [889ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [893ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@552497245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@701c018c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774) -> (continuation of Deferred@832393612 after retry RPC@1660492861) -> (continuation of Deferred@2107830440 after retry RPC@1460667349), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79590be4 -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774) -> (continuation of Deferred@832393612 after retry RPC@1660492861) -> (continuation of Deferred@2107830440 after retry RPC@1460667349))) after lookup
06:12:59.061 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:12:59.062 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:59.062 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:59.063 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:12:59.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.117579  7913 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:12:59.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.265601  7924 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:12:59.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.266022  7924 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:59.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.268380  7924 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:59.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:59.272473  7557 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:12:59.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:59.276688  7557 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:12:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.279423  7855 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6" is_pre_election: true
06:12:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.280237  7855 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 3.
06:12:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.281481  7557 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:12:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.282167  7924 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 4 FOLLOWER]: Leader pre-election won for term 5
06:12:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.282475  7924 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 4 FOLLOWER]: Starting leader election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:12:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.282701  7924 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 4 FOLLOWER]: Advancing to term 5
06:12:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.287191  7924 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.288709  7924 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 5 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:12:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.289489  7855 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6"
06:12:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.290042  7855 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 3 FOLLOWER]: Advancing to term 5
06:12:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:59.292182  7557 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:12:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.299098  7855 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 5.
06:12:59.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.300333  7557 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:12:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.300936  7924 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 5 FOLLOWER]: Leader election won for term 5
06:12:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.306041  7924 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 5 LEADER]: Becoming Leader. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Running, Role: LEADER
06:12:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.306908  7924 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:12:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.316085  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 reported cstate change: term changed from 4 to 5, leader changed from e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131) to 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129). New cstate: current_term: 5 leader_uuid: "08e4006c4cde42189b7529a3c7455cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: HEALTHY } } }
06:12:59.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.736768  7855 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 5 FOLLOWER]: Refusing update from remote peer 08e4006c4cde42189b7529a3c7455cc4: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
06:12:59.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.738425  7924 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
06:12:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:12:59.753448  7951 mvcc.cc:204] Tried to move back new op lower bound from 7317566161263812608 to 7317566158182273024. Current Snapshot: MvccSnapshot[applied={T|T < 7317566161263812608}]
06:12:59.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:12:59.775415  7557 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 -> Peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Couldn't send request to peer e93dadc87c1d4f888622d646f01f4da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:13:00.439 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:00.441 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:00.441 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=2274), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [8ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [12ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [13ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [15ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [15ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [33ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [34ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [36ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [36ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [69ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [72ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [74ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [76ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [76ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [209ms] refreshing cache from master, [209ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [213ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [214ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [216ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [216ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [629ms] refreshing cache from master, [630ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [633ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [635ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [636ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [637ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [889ms] refreshing cache from master, [889ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [893ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [894ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [896ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [897ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2269ms] refreshing cache from master, [2269ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [2273ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@552497245(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@701c018c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774) -> (continuation of Deferred@832393612 after retry RPC@1660492861) -> (continuation of Deferred@2107830440 after retry RPC@1460667349) -> (continuation of Deferred@224841320 after retry RPC@1522494138), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@79590be4 -> wakeup thread Time-limited test -> (continuation of Deferred@297724557 after retry RPC@545899893) -> (continuation of Deferred@1793243658 after retry RPC@70150222) -> (continuation of Deferred@13629473 after retry RPC@2002033774) -> (continuation of Deferred@832393612 after retry RPC@1660492861) -> (continuation of Deferred@2107830440 after retry RPC@1460667349) -> (continuation of Deferred@224841320 after retry RPC@1522494138))) after lookup
06:13:00.442 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:00.443 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] Successfully connected to peer
06:13:00.463 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x7470c857, L:/127.0.0.1:59374 - R:/127.6.39.129:41443] using TOKEN/null
06:13:00.508 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.131:35835
06:13:00.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:00.510828  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:13:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:00.822393  7969 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:13:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:00.822950  7969 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:13:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:00.823601  7969 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:13:00.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:00.856444  7969 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:13:00.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:00.857559  7969 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.131
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:00.894603  7969 tablet_server_runner.cc:78] Tablet server non-default flags:
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:13:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:00.896364  7969 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:13:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:00.898124  7969 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:13:00.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:00.911152  7980 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:02.066412  7557 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 -> Peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Couldn't send request to peer e93dadc87c1d4f888622d646f01f4da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:13:02.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:00.911557  7982 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:02.277529  7969 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.366s	user 0.534s	sys 0.829s
06:13:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:02.277660  7983 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1365 milliseconds
06:13:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:02.277765  7984 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:02.277885  7969 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.367s	user 0.534s	sys 0.829s
06:13:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.278237  7969 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:13:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.279723  7969 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:13:02.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.282603  7969 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:13:02.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.284058  7969 hybrid_clock.cc:648] HybridClock initialized: now 1786515182283897 us; error 146 us; skew 500 ppm
06:13:02.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.285138  7969 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:13:02.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.293586  7969 webserver.cc:533] Webserver started at http://127.6.39.131:41543/ using document root <none> and password file <none>
06:13:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.294615  7969 fs_manager.cc:362] Metadata directory not provided
06:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.294848  7969 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:13:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.303260  7969 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
06:13:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.308106  7993 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:13:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.309434  7969 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
06:13:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.309810  7969 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:13:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:42 on dist-test-slave-vq2q"
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.312525  7969 fs_report.cc:389] FS layout report
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/data
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:13:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:13:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.399722  7969 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:13:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.401965  7969 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:13:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.402602  7969 kserver.cc:163] Server-wide thread pool size limit: 3276
06:13:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.406520  7969 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:13:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.413645  8000 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:13:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.422004  7969 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:13:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.422228  7969 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
06:13:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.422506  7969 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:13:02.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.427129  7969 ts_tablet_manager.cc:616] Registered 1 tablets
06:13:02.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.427305  7969 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.001s	sys 0.003s
06:13:02.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.427778  8000 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap starting.
06:13:02.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.485211  8000 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Log is configured to *not* fsync() on all Append() calls
06:13:02.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.545006  8000 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:13:02.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.545943  8000 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap complete.
06:13:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.547294  8000 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent bootstrapping tablet: real 0.120s	user 0.090s	sys 0.024s
06:13:02.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.568483  8000 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.574935  8000 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Initialized, Role: FOLLOWER
06:13:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.577225  8000 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:02.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.590003  8000 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent starting tablet: real 0.042s	user 0.031s	sys 0.009s
06:13:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.619376  7969 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.131:35835
06:13:02.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.619607  8130 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.131:35835 every 8 connection(s)
06:13:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.622195  7969 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:02.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.628618  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 7969
06:13:02.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:02.649 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:02.650 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.129:41443
06:13:02.651 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:13:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.651672  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 7528
06:13:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.671003  8132 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:13:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.671558  8132 heartbeater.cc:461] Registering TS with master...
06:13:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.672930  8132 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:13:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.677307  6533 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.677883  8133 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:13:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.678222  8133 heartbeater.cc:461] Registering TS with master...
06:13:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.679350  8133 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:13:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.680033  6533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.131:39801
06:13:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.683383  6446 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:02.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.684687  8134 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:13:02.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.685047  8134 heartbeater.cc:461] Registering TS with master...
06:13:02.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.685783  8134 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:13:02.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:02.688877  6353 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:02.698 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] handling channelInactive
06:13:02.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state READY due to: connection closed
06:13:02.698 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:02.699 [DEBUG - Time-limited test] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:02.700 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.700 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.700 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.719 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:02.720 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:02.720 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1159239234(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@384124b9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@aeec2f7 -> wakeup thread Time-limited test)) after lookup
06:13:02.721 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:02.722 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.722 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.722 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.738 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:02.738 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:02.739 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [23ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [23ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1159239234(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@384124b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@aeec2f7 -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227))) after lookup
06:13:02.739 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:02.741 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.741 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.742 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.759 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:02.759 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:02.759 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [23ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [23ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [40ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [43ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [43ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [59ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1159239234(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@384124b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@aeec2f7 -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864))) after lookup
06:13:02.760 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:02.761 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.761 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.761 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.819 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:02.819 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:02.819 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=120), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [23ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [23ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [40ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [43ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [43ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [59ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [61ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [62ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [62ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [119ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1159239234(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@384124b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864) -> (continuation of Deferred@356510368 after retry RPC@109742986), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@aeec2f7 -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864) -> (continuation of Deferred@356510368 after retry RPC@109742986))) after lookup
06:13:02.820 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:02.821 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.821 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:02.821 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:03.199 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:03.199 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:03.200 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=501), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [23ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [23ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [40ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [43ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [43ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [59ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [61ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [62ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [62ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [119ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [120ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [122ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [122ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [496ms] refreshing cache from master, [496ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [500ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1159239234(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@384124b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864) -> (continuation of Deferred@356510368 after retry RPC@109742986) -> (continuation of Deferred@723967852 after retry RPC@2040711001), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@aeec2f7 -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864) -> (continuation of Deferred@356510368 after retry RPC@109742986) -> (continuation of Deferred@723967852 after retry RPC@2040711001))) after lookup
06:13:03.200 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:03.201 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:03.202 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:03.202 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.685981  8132 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:13:03.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.850148  8147 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 08e4006c4cde42189b7529a3c7455cc4)
06:13:03.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.850567  8147 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:03.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.852860  8147 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:03.857093  7768 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:03.861763  7768 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.865025  8075 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "7a785279bacd40deb619d9f334289da6" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "e93dadc87c1d4f888622d646f01f4da6" is_pre_election: true
06:13:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.865890  8075 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7a785279bacd40deb619d9f334289da6 in term 4.
06:13:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.867611  7774 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4
06:13:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.868417  8147 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 5 FOLLOWER]: Leader pre-election won for term 6
06:13:03.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.868738  8147 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 08e4006c4cde42189b7529a3c7455cc4)
06:13:03.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.869056  8147 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 5 FOLLOWER]: Advancing to term 6
06:13:03.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.875893  8147 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:03.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.877952  8147 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 6 election: Requested vote from peers e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:03.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.878736  8075 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "7a785279bacd40deb619d9f334289da6" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:13:03.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.879325  8075 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 4 FOLLOWER]: Advancing to term 6
06:13:03.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:03.882925  7768 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:03.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.886639  8075 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7a785279bacd40deb619d9f334289da6 in term 6.
06:13:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.887697  7774 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4
06:13:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.888375  8147 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 6 FOLLOWER]: Leader election won for term 6
06:13:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.890882  8147 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 6 LEADER]: Becoming Leader. State: Replica: 7a785279bacd40deb619d9f334289da6, State: Running, Role: LEADER
06:13:03.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.892304  8147 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:03.902431  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 reported cstate change: term changed from 5 to 6, leader changed from 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129) to 7a785279bacd40deb619d9f334289da6 (127.6.39.130). New cstate: current_term: 6 leader_uuid: "7a785279bacd40deb619d9f334289da6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: UNKNOWN } } }
06:13:04.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:04.411790  8075 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 6 FOLLOWER]: Refusing update from remote peer 7a785279bacd40deb619d9f334289da6: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
06:13:04.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:04.413492  8166 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
06:13:04.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:04.427983  7768 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 -> Peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Couldn't send request to peer 08e4006c4cde42189b7529a3c7455cc4. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:13:05.020 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:05.020 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:05.021 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2322), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [23ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [23ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [39ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [40ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [43ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [43ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [59ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [61ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [62ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [62ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [116ms] refreshing cache from master, [116ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [119ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [120ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [122ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [122ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [496ms] refreshing cache from master, [496ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [500ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [501ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [503ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [503ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [2316ms] refreshing cache from master, [2316ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [2321ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1159239234(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@384124b9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864) -> (continuation of Deferred@356510368 after retry RPC@109742986) -> (continuation of Deferred@723967852 after retry RPC@2040711001) -> (continuation of Deferred@159423860 after retry RPC@1727470354), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@aeec2f7 -> wakeup thread Time-limited test -> (continuation of Deferred@1945929729 after retry RPC@267978227) -> (continuation of Deferred@318157644 after retry RPC@2005697864) -> (continuation of Deferred@356510368 after retry RPC@109742986) -> (continuation of Deferred@723967852 after retry RPC@2040711001) -> (continuation of Deferred@159423860 after retry RPC@1727470354))) after lookup
06:13:05.021 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:05.023 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] Successfully connected to peer
06:13:05.045 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xa2ca2a5f, L:/127.0.0.1:60648 - R:/127.6.39.130:42467] using TOKEN/null
06:13:05.103 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.129:41443
06:13:05.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:05.106745  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:41443
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45273
06:13:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:13:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:13:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:05.414901  8194 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:13:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:05.415465  8194 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:13:05.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:05.415939  8194 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:13:05.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:05.449316  8194 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:13:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:05.450578  8194 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.129
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:05.488880  8194 tablet_server_runner.cc:78] Tablet server non-default flags:
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:41443
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45273
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:13:05.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:13:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:13:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:05.490800  8194 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:13:05.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:05.492370  8194 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:13:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:05.504267  8201 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:06.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:05.506009  8202 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:06.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:06.845515  8203 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1338 milliseconds
06:13:06.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:06.846362  8205 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:06.849263  8194 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.344s	user 0.468s	sys 0.839s
06:13:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:06.849521  8194 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.345s	user 0.468s	sys 0.842s
06:13:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.849715  8194 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:13:06.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.851877  8194 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:13:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.854133  8194 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:13:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.855525  8194 hybrid_clock.cc:648] HybridClock initialized: now 1786515186855464 us; error 69 us; skew 500 ppm
06:13:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.856426  8194 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:13:06.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.864856  8194 webserver.cc:533] Webserver started at http://127.6.39.129:45273/ using document root <none> and password file <none>
06:13:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.865880  8194 fs_manager.cc:362] Metadata directory not provided
06:13:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.866108  8194 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:13:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.880082  8194 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.000s	sys 0.009s
06:13:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.886070  8217 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:13:06.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.888298  8194 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.001s	sys 0.003s
06:13:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.888679  8194 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:13:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:38 on dist-test-slave-vq2q"
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.891351  8194 fs_report.cc:389] FS layout report
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/data
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:13:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:13:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.963724  8194 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:13:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.965245  8194 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:13:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.965646  8194 kserver.cc:163] Server-wide thread pool size limit: 3276
06:13:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.968194  8194 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:13:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.973901  8227 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:13:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.982090  8194 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:13:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.982316  8194 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
06:13:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.982530  8194 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:13:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.987272  8194 ts_tablet_manager.cc:616] Registered 1 tablets
06:13:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.987484  8194 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
06:13:06.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:06.988061  8227 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap starting.
06:13:07.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.048713  8227 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Log is configured to *not* fsync() on all Append() calls
06:13:07.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.122403  8227 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:13:07.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.123555  8227 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap complete.
06:13:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.125627  8227 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent bootstrapping tablet: real 0.138s	user 0.095s	sys 0.040s
06:13:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:07.152487  7768 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 -> Peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Couldn't send request to peer 08e4006c4cde42189b7529a3c7455cc4. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:13:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.152171  8227 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.154845  8227 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Initialized, Role: FOLLOWER
06:13:07.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.155866  8227 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.159693  8227 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent starting tablet: real 0.034s	user 0.026s	sys 0.006s
06:13:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.189049  8194 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.129:41443
06:13:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.189396  8362 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.129:41443 every 8 connection(s)
06:13:07.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.191530  8194 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:13:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.198172  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 8194
06:13:07.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:07.229 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:07.230 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.130:42467
06:13:07.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:13:07.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.231369  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 7739
06:13:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.249789  8364 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:13:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.250277  8364 heartbeater.cc:461] Registering TS with master...
06:13:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.251627  8364 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:13:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.255630  8363 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:13:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.256227  8363 heartbeater.cc:461] Registering TS with master...
06:13:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.257326  8363 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:13:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.260385  6446 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.261317  6533 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:07.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.264179  6533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.129:35947
06:13:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.272182  8365 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:13:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.272543  8365 heartbeater.cc:461] Registering TS with master...
06:13:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.273372  8365 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:13:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:07.276683  6353 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:07.283 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] handling channelInactive
06:13:07.283 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state READY due to: connection closed
06:13:07.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:07.285 [DEBUG - Time-limited test] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:07.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.286 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.299 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:07.299 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:07.299 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1992839703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2839e857 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61b49b61 -> wakeup thread Time-limited test)) after lookup
06:13:07.300 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:07.301 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.301 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.301 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.319 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:07.319 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:07.319 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [17ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [17ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1992839703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2839e857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61b49b61 -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825))) after lookup
06:13:07.320 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:07.321 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.321 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.321 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.359 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:07.359 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:07.359 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [17ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [17ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [37ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [37ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1992839703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2839e857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61b49b61 -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342))) after lookup
06:13:07.360 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:07.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.362 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.439 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:07.439 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:07.439 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=154), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [17ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [17ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [37ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [37ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [77ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [78ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1992839703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2839e857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61b49b61 -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175))) after lookup
06:13:07.440 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:07.441 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.441 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.441 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.559 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:07.560 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:07.560 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=275), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [17ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [17ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [37ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [37ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [77ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [78ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [156ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [157ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [157ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [271ms] refreshing cache from master, [271ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [275ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1992839703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2839e857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175) -> (continuation of Deferred@1451131402 after retry RPC@1479182617), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61b49b61 -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175) -> (continuation of Deferred@1451131402 after retry RPC@1479182617))) after lookup
06:13:07.561 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:07.562 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.562 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:07.562 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.267815  8363 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:13:08.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.494526  8373 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 7a785279bacd40deb619d9f334289da6)
06:13:08.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.494941  8373 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:08.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.498211  8373 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:08.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:08.500968  7997 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:08.502 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:08.503 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:08.503 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1218), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [17ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [17ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [37ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [37ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [77ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [78ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [156ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [157ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [157ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [271ms] refreshing cache from master, [271ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [275ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [277ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [278ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [278ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1211ms] refreshing cache from master, [1211ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1218ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1992839703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2839e857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175) -> (continuation of Deferred@1451131402 after retry RPC@1479182617) -> (continuation of Deferred@394140871 after retry RPC@454045040), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61b49b61 -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175) -> (continuation of Deferred@1451131402 after retry RPC@1479182617) -> (continuation of Deferred@394140871 after retry RPC@454045040))) after lookup
06:13:08.504 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:08.505 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:08.505 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:08.506 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:08.507297  7997 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.512454  8300 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4" is_pre_election: true
06:13:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.513271  8300 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 5.
06:13:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.514675  7994 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, e93dadc87c1d4f888622d646f01f4da6; no voters: 7a785279bacd40deb619d9f334289da6
06:13:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.515494  8373 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 6 FOLLOWER]: Leader pre-election won for term 7
06:13:08.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.515753  8373 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 7a785279bacd40deb619d9f334289da6)
06:13:08.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.515995  8373 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 6 FOLLOWER]: Advancing to term 7
06:13:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.520399  8373 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:08.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.521872  8373 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 7 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:08.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.523028  8300 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:13:08.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.523581  8300 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 5 FOLLOWER]: Advancing to term 7
06:13:08.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:08.525126  7997 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.530134  8300 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 7.
06:13:08.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.531018  7994 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, e93dadc87c1d4f888622d646f01f4da6; no voters: 7a785279bacd40deb619d9f334289da6
06:13:08.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.531641  8373 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Leader election won for term 7
06:13:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.533243  8373 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 LEADER]: Becoming Leader. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Running, Role: LEADER
06:13:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.534228  8373 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:08.542835  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 reported cstate change: term changed from 6 to 7, leader changed from 7a785279bacd40deb619d9f334289da6 (127.6.39.130) to e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131). New cstate: current_term: 7 leader_uuid: "e93dadc87c1d4f888622d646f01f4da6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: UNKNOWN } } }
06:13:08.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:08.943784  7997 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 -> Peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Couldn't send request to peer 7a785279bacd40deb619d9f334289da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:13:09.099 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:09.100 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:09.100 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=14, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1815), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [11ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [17ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [17ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [35ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [36ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [37ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [37ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [71ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [75ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [77ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [78ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [151ms] refreshing cache from master, [151ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [154ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [156ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [157ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [157ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [271ms] refreshing cache from master, [271ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [275ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [277ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [278ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [278ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1211ms] refreshing cache from master, [1211ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1218ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [1220ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [1222ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1222ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1811ms] refreshing cache from master, [1811ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1815ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1992839703(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2839e857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175) -> (continuation of Deferred@1451131402 after retry RPC@1479182617) -> (continuation of Deferred@394140871 after retry RPC@454045040) -> (continuation of Deferred@311128316 after retry RPC@1622242187), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@61b49b61 -> wakeup thread Time-limited test -> (continuation of Deferred@1475106539 after retry RPC@768756825) -> (continuation of Deferred@182101720 after retry RPC@903189342) -> (continuation of Deferred@605047608 after retry RPC@1028023175) -> (continuation of Deferred@1451131402 after retry RPC@1479182617) -> (continuation of Deferred@394140871 after retry RPC@454045040) -> (continuation of Deferred@311128316 after retry RPC@1622242187))) after lookup
06:13:09.101 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:09.102 [DEBUG - kudu-nio-0] (Connection.java:757) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] Successfully connected to peer
06:13:09.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:09.107977  8300 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 7 FOLLOWER]: Refusing update from remote peer e93dadc87c1d4f888622d646f01f4da6: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
06:13:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:09.109545  8373 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
06:13:09.136 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xc0fa2c98, L:/127.0.0.1:50680 - R:/127.6.39.131:35835] using TOKEN/null
06:13:09.221 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.130:42467
06:13:09.222 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:09.223959  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:42467
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42227
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:13:09.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:09.540434  8408 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:13:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:09.540921  8408 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:13:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:09.541525  8408 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:13:09.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:09.574144  8408 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:13:09.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:09.575178  8408 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.130
06:13:09.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:09.611116  8408 tablet_server_runner.cc:78] Tablet server non-default flags:
06:13:09.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:09.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:09.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:09.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:13:09.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:13:09.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:42467
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42227
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:13:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:13:09.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:09.612876  8408 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:13:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:09.614691  8408 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:13:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:09.627561  8419 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:10.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:09.628854  8420 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:10.932739  8422 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:10.933334  8408 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.304s	user 0.509s	sys 0.730s
06:13:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:10.933303  8421 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1301 milliseconds
06:13:10.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:10.933691  8408 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.305s	user 0.513s	sys 0.730s
06:13:10.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.934056  8408 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:13:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.935400  8408 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:13:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.942086  8408 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:13:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.943544  8408 hybrid_clock.cc:648] HybridClock initialized: now 1786515190943493 us; error 63 us; skew 500 ppm
06:13:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.944379  8408 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:13:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.951997  8408 webserver.cc:533] Webserver started at http://127.6.39.130:42227/ using document root <none> and password file <none>
06:13:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.952893  8408 fs_manager.cc:362] Metadata directory not provided
06:13:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.953089  8408 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:13:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.961967  8408 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.004s	sys 0.004s
06:13:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.967183  8435 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:13:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.968616  8408 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.001s	sys 0.005s
06:13:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.969045  8408 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a785279bacd40deb619d9f334289da6"
06:13:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:40 on dist-test-slave-vq2q"
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:10.971810  8408 fs_report.cc:389] FS layout report
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/data
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:13:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:13:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:13:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:13:11.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.060536  8408 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:13:11.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.062456  8408 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:13:11.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.063019  8408 kserver.cc:163] Server-wide thread pool size limit: 3276
06:13:11.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.066298  8408 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:13:11.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.071920  8445 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:13:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.080241  8408 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:13:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.080488  8408 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
06:13:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.080721  8408 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:13:11.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.085631  8408 ts_tablet_manager.cc:616] Registered 1 tablets
06:13:11.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.085898  8408 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
06:13:11.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.086351  8445 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap starting.
06:13:11.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.169742  8445 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Log is configured to *not* fsync() on all Append() calls
06:13:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.267287  8445 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:13:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.268407  8445 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap complete.
06:13:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.269824  8445 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent bootstrapping tablet: real 0.184s	user 0.115s	sys 0.056s
06:13:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.287447  8445 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:11.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.289683  8445 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7a785279bacd40deb619d9f334289da6, State: Initialized, Role: FOLLOWER
06:13:11.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.290704  8445 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.295185  8445 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent starting tablet: real 0.025s	user 0.020s	sys 0.004s
06:13:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.295642  8408 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.130:42467
06:13:11.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.295926  8577 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.130:42467 every 8 connection(s)
06:13:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.298317  8408 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:13:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.299471  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 8408
06:13:11.300 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:11.338 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:11.339 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.131:35835
06:13:11.340 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:13:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.341044  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 7969
06:13:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.355803  8580 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:13:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.356333  8580 heartbeater.cc:461] Registering TS with master...
06:13:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.357672  8579 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:13:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.358068  8580 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:13:11.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.358153  8579 heartbeater.cc:461] Registering TS with master...
06:13:11.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.360000  8579 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:13:11.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.363287  6533 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:13:11.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.363417  6446 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:13:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.365501  8581 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:13:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.365830  8581 heartbeater.cc:461] Registering TS with master...
06:13:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.366271  6533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.130:52757
06:13:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.366761  8581 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:13:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:11.370617  6354 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:13:11.384 [DEBUG - kudu-nio-0] (Connection.java:239) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] handling channelInactive
06:13:11.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state READY due to: connection closed
06:13:11.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:11.386 [DEBUG - Time-limited test] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:11.387 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.387 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.387 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.400 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:11.400 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:11.400 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [15ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@868664489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5880c471 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@53fb7d54 -> wakeup thread Time-limited test)) after lookup
06:13:11.401 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:11.402 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.402 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.402 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.419 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:11.419 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:11.420 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [15ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [17ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [17ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@868664489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5880c471 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@53fb7d54 -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074))) after lookup
06:13:11.420 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:11.421 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.421 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.422 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.439 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:11.440 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:11.440 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=54), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [15ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [17ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [17ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@868664489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5880c471 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@53fb7d54 -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496))) after lookup
06:13:11.441 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:11.442 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.442 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.442 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.539 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:11.539 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:11.539 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=153), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [15ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [17ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [17ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [55ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [57ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [57ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [153ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@868664489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5880c471 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@53fb7d54 -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832))) after lookup
06:13:11.540 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:11.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.541 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.679 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:11.679 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:11.680 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=294), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [15ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [17ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [17ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [55ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [57ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [57ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [153ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [154ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [156ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [156ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [290ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [294ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@868664489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5880c471 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832) -> (continuation of Deferred@1164409059 after retry RPC@141498197), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@53fb7d54 -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832) -> (continuation of Deferred@1164409059 after retry RPC@141498197))) after lookup
06:13:11.680 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:11.682 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.682 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:11.682 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:12.220 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:12.221 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:12.221 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=835), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [15ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [17ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [17ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [55ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [57ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [57ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [153ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [154ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [156ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [156ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [290ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [294ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [295ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [297ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [297ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [831ms] refreshing cache from master, [831ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [835ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@868664489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5880c471 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832) -> (continuation of Deferred@1164409059 after retry RPC@141498197) -> (continuation of Deferred@1458476681 after retry RPC@642512574), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@53fb7d54 -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832) -> (continuation of Deferred@1164409059 after retry RPC@141498197) -> (continuation of Deferred@1458476681 after retry RPC@642512574))) after lookup
06:13:12.222 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:12.223 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:12.223 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:12.223 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:12.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.371603  8579 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:13:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.554975  8592 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:13:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.555475  8592 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:12.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.558059  8592 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:12.562451  8223 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:13:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:12.566946  8223 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:13:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.569801  8527 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6" is_pre_election: true
06:13:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.570456  8527 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 6.
06:13:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.571624  8223 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:13:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.572237  8592 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 7 FOLLOWER]: Leader pre-election won for term 8
06:13:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.572535  8592 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 7 FOLLOWER]: Starting leader election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:13:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.572765  8592 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 7 FOLLOWER]: Advancing to term 8
06:13:12.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.577289  8592 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:12.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.578789  8592 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 8 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.579680  8527 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6"
06:13:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.580133  8527 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 6 FOLLOWER]: Advancing to term 8
06:13:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:12.582460  8223 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:13:12.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.587335  8527 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 8.
06:13:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.588366  8223 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:13:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.589110  8592 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Leader election won for term 8
06:13:12.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.590902  8592 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 LEADER]: Becoming Leader. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Running, Role: LEADER
06:13:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.591842  8592 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:12.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:12.600208  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 reported cstate change: term changed from 7 to 8, leader changed from e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131) to 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129). New cstate: current_term: 8 leader_uuid: "08e4006c4cde42189b7529a3c7455cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: HEALTHY } } }
06:13:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:13.016610  8527 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 8 FOLLOWER]: Refusing update from remote peer 08e4006c4cde42189b7529a3c7455cc4: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
06:13:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:13.018127  8592 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
06:13:13.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:13.048525  8223 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 -> Peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Couldn't send request to peer e93dadc87c1d4f888622d646f01f4da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:13:14.460 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:14.461 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:14.461 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=3075), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [2ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [2ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [15ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [16ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [17ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [17ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [37ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [37ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [50ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [54ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [55ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [57ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [57ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [150ms] refreshing cache from master, [150ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [153ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [154ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [156ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [156ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [290ms] refreshing cache from master, [290ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [294ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [295ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [297ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [297ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [831ms] refreshing cache from master, [831ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [835ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [837ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [838ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [838ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3070ms] refreshing cache from master, [3071ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [3075ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@868664489(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5880c471 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832) -> (continuation of Deferred@1164409059 after retry RPC@141498197) -> (continuation of Deferred@1458476681 after retry RPC@642512574) -> (continuation of Deferred@1516236110 after retry RPC@601366205), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@53fb7d54 -> wakeup thread Time-limited test -> (continuation of Deferred@1915637521 after retry RPC@1455176074) -> (continuation of Deferred@272178380 after retry RPC@670963496) -> (continuation of Deferred@1036318207 after retry RPC@1783947832) -> (continuation of Deferred@1164409059 after retry RPC@141498197) -> (continuation of Deferred@1458476681 after retry RPC@642512574) -> (continuation of Deferred@1516236110 after retry RPC@601366205))) after lookup
06:13:14.462 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:14.463 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] Successfully connected to peer
06:13:14.482 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xfe7abbe9, L:/127.0.0.1:53918 - R:/127.6.39.129:41443] using TOKEN/null
06:13:14.529 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.131:35835
06:13:14.530 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:14.531183  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:13:15.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.272182  8638 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:13:15.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.273280  8638 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:13:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.274549  8638 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:13:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.396239  8638 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:13:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.397872  8638 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.131
06:13:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:15.472165  8638 tablet_server_runner.cc:78] Tablet server non-default flags:
06:13:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:13:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:15.476048  8638 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:13:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:15.479251  8638 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:13:15.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.505540  8651 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.605392  8223 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 -> Peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Couldn't send request to peer e93dadc87c1d4f888622d646f01f4da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:13:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:17.531512  8638 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.023s	user 0.723s	sys 1.152s
06:13:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:16.924342  8650 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 8638
06:13:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:17.531926  8638 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.024s	user 0.724s	sys 1.152s
06:13:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:15.507925  8653 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:17.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:17.534130  8655 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:17.536866  8654 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2022 milliseconds
06:13:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.536897  8638 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:13:17.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.538273  8638 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:13:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.540373  8638 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:13:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.541702  8638 hybrid_clock.cc:648] HybridClock initialized: now 1786515197541665 us; error 45 us; skew 500 ppm
06:13:17.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.542620  8638 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:13:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.549309  8638 webserver.cc:533] Webserver started at http://127.6.39.131:41543/ using document root <none> and password file <none>
06:13:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.550262  8638 fs_manager.cc:362] Metadata directory not provided
06:13:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.550457  8638 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:13:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.558812  8638 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.000s
06:13:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.563530  8669 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:13:17.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.564671  8638 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.565021  8638 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:42 on dist-test-slave-vq2q"
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.567029  8638 fs_report.cc:389] FS layout report
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/data
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:13:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:13:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:13:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:13:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:13:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:13:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:13:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:13:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:13:17.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.639292  8638 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:13:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.640998  8638 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:13:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.641577  8638 kserver.cc:163] Server-wide thread pool size limit: 3276
06:13:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.644848  8638 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:13:17.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.652024  8679 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:13:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.661975  8638 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:13:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.662228  8638 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.003s	sys 0.000s
06:13:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.662571  8638 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:13:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.667558  8638 ts_tablet_manager.cc:616] Registered 1 tablets
06:13:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.667747  8638 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
06:13:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.668160  8679 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap starting.
06:13:17.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.741357  8679 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Log is configured to *not* fsync() on all Append() calls
06:13:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:17.819448  8223 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111) [suppressed 10 similar messages]
06:13:17.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.833040  8679 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:13:17.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.834075  8679 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap complete.
06:13:17.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.835649  8679 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent bootstrapping tablet: real 0.168s	user 0.098s	sys 0.063s
06:13:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.853034  8679 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.855818  8679 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Initialized, Role: FOLLOWER
06:13:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.857038  8679 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.864919  8679 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent starting tablet: real 0.029s	user 0.022s	sys 0.007s
06:13:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.885771  8638 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.131:35835
06:13:17.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.886698  8804 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.131:35835 every 8 connection(s)
06:13:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.889479  8638 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:17.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.892584  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 8638
06:13:17.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:17.940 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:17.941 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.129:41443
06:13:17.941 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:13:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.942319  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 8194
06:13:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.985424  8807 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:13:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.985611  8806 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:13:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.986054  8807 heartbeater.cc:461] Registering TS with master...
06:13:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.986052  8806 heartbeater.cc:461] Registering TS with master...
06:13:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.987645  8807 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:13:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.987732  8806 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:13:17.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.995004  6354 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:17.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:17.995126  6446 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:18.000798  8805 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:13:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:18.001195  8805 heartbeater.cc:461] Registering TS with master...
06:13:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:18.002216  8805 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:13:18.004 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] handling channelInactive
06:13:18.004 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state READY due to: connection closed
06:13:18.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:18.006 [DEBUG - Time-limited test] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:18.007 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.008 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:18.008018  6535 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:18.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:18.011787  6535 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.131:48729
06:13:18.019 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:18.020 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:18.020 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [3ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@956295773(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5b313609 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b91e673 -> wakeup thread Time-limited test)) after lookup
06:13:18.021 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:18.022 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.022 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.022 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.039 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:18.040 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:18.040 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=34), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [3ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [15ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [17ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [17ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@956295773(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5b313609 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b91e673 -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300))) after lookup
06:13:18.041 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:18.042 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.042 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.042 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.080 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:18.080 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:18.080 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [3ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [15ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [17ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [17ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [74ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@956295773(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5b313609 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b91e673 -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488))) after lookup
06:13:18.081 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:18.083 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.083 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.083 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.199 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:18.200 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:18.200 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=194), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [3ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [15ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [17ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [17ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [74ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [78ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [78ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [194ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@956295773(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5b313609 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b91e673 -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378))) after lookup
06:13:18.201 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:18.202 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.202 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.203 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.378 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:18.379 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:18.379 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=374), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [3ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [15ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [17ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [17ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [74ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [78ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [78ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [194ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [195ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [198ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [198ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [370ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [373ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@956295773(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5b313609 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378) -> (continuation of Deferred@557258665 after retry RPC@1761104369), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b91e673 -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378) -> (continuation of Deferred@557258665 after retry RPC@1761104369))) after lookup
06:13:18.380 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:18.381 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.381 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:18.382 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:19.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.016099  8805 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:13:19.079 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:19.079 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:19.080 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1074), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [3ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [15ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [17ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [17ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [74ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [78ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [78ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [194ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [195ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [198ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [198ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [370ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [373ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [375ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [376ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [377ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1070ms] refreshing cache from master, [1070ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1074ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@956295773(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5b313609 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378) -> (continuation of Deferred@557258665 after retry RPC@1761104369) -> (continuation of Deferred@218970442 after retry RPC@667981507), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b91e673 -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378) -> (continuation of Deferred@557258665 after retry RPC@1761104369) -> (continuation of Deferred@218970442 after retry RPC@667981507))) after lookup
06:13:19.080 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:19.082 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:19.082 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state CONNECTING due to: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:19.082 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443
06:13:19.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.349027  8822 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:13:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.349601  8822 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.352085  8822 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:19.357247  8670 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:19.361517  8670 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.365077  8527 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6" is_pre_election: true
06:13:19.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.366772  8673 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6
06:13:19.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.367486  8822 raft_consensus.cc:2749] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
06:13:19.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.391697  8828 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 08e4006c4cde42189b7529a3c7455cc4)
06:13:19.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.392141  8828 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.395143  8828 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:19.399755  8436 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:19.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:19.404244  8436 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.408140  8750 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "7a785279bacd40deb619d9f334289da6" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "e93dadc87c1d4f888622d646f01f4da6" is_pre_election: true
06:13:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.408902  8750 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7a785279bacd40deb619d9f334289da6 in term 7.
06:13:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.410372  8440 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4
06:13:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.411257  8828 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 8 FOLLOWER]: Leader pre-election won for term 9
06:13:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.411545  8828 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 08e4006c4cde42189b7529a3c7455cc4)
06:13:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.411774  8828 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 8 FOLLOWER]: Advancing to term 9
06:13:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.417287  8828 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:19.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.418797  8828 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 9 election: Requested vote from peers e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.419763  8750 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "7a785279bacd40deb619d9f334289da6" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:13:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.420320  8750 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 7 FOLLOWER]: Advancing to term 9
06:13:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:19.422657  8436 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111)
06:13:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.427114  8750 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7a785279bacd40deb619d9f334289da6 in term 9.
06:13:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.428036  8440 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6; no voters: 08e4006c4cde42189b7529a3c7455cc4
06:13:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.428658  8828 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 9 FOLLOWER]: Leader election won for term 9
06:13:19.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.430166  8828 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 9 LEADER]: Becoming Leader. State: Replica: 7a785279bacd40deb619d9f334289da6, State: Running, Role: LEADER
06:13:19.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.431445  8828 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:19.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.440534  6535 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 reported cstate change: term changed from 8 to 9, leader changed from 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129) to 7a785279bacd40deb619d9f334289da6 (127.6.39.130). New cstate: current_term: 9 leader_uuid: "7a785279bacd40deb619d9f334289da6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: UNKNOWN } } }
06:13:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:19.945393  8436 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 -> Peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Couldn't send request to peer 08e4006c4cde42189b7529a3c7455cc4. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:13:19.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.951031  8750 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 9 FOLLOWER]: Refusing update from remote peer 7a785279bacd40deb619d9f334289da6: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
06:13:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:19.952749  8855 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.001s
06:13:22.039 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:22.039 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:22.040 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=4034), Traces: [0ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [3ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [3ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [14ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [15ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [17ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [17ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [30ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [34ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [35ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [37ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [37ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [74ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [76ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [78ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [78ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [190ms] refreshing cache from master, [190ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [194ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [195ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [198ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [198ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [370ms] refreshing cache from master, [370ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [373ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [375ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [376ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [377ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1070ms] refreshing cache from master, [1070ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1074ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [1075ms] sending RPC to server 08e4006c4cde42189b7529a3c7455cc4, [1077ms] received response from server 08e4006c4cde42189b7529a3c7455cc4: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [1077ms] delaying RPC due to: Network error: Failed to connect to peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443): Connection refused: /127.6.39.129:41443, [4030ms] refreshing cache from master, [4030ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [4034ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@956295773(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5b313609 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378) -> (continuation of Deferred@557258665 after retry RPC@1761104369) -> (continuation of Deferred@218970442 after retry RPC@667981507) -> (continuation of Deferred@2143852468 after retry RPC@1552773120), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5b91e673 -> wakeup thread Time-limited test -> (continuation of Deferred@1130546524 after retry RPC@1868638300) -> (continuation of Deferred@1478589952 after retry RPC@1046777488) -> (continuation of Deferred@1806505921 after retry RPC@1898382378) -> (continuation of Deferred@557258665 after retry RPC@1761104369) -> (continuation of Deferred@218970442 after retry RPC@667981507) -> (continuation of Deferred@2143852468 after retry RPC@1552773120))) after lookup
06:13:22.040 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:22.041 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] Successfully connected to peer
06:13:22.062 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xbc738d5d, L:/127.0.0.1:32830 - R:/127.6.39.130:42467] using TOKEN/null
06:13:22.109 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.129:41443
06:13:22.110 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:22.112183  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:13:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:41443
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45273
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:13:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.426631  8889 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:13:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.427172  8889 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:13:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.427786  8889 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:13:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.441232  8436 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 -> Peer 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443): Couldn't send request to peer 08e4006c4cde42189b7529a3c7455cc4. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.129:41443: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:13:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.461844  8889 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:13:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.462674  8889 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.129
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:22.497747  8889 tablet_server_runner.cc:78] Tablet server non-default flags:
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.129:41443
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.129
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45273
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.129
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/logs
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:13:22.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:22.499199  8889 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:13:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:22.501477  8889 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:13:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.517851  8897 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:23.920382  8896 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 8889
06:13:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:23.960896  8896 kernel_stack_watchdog.cc:198] Thread 8889 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
06:13:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
06:13:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
06:13:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
06:13:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
06:13:23.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:23.963832  8901 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:23.964470  8889 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.444s	user 0.532s	sys 0.875s
06:13:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:23.965124  8889 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.447s	user 0.532s	sys 0.876s
06:13:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:22.518405  8899 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.969877  8889 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:13:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:23.969800  8900 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1450 milliseconds
06:13:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.971778  8889 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:13:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.974190  8889 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:13:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.975533  8889 hybrid_clock.cc:648] HybridClock initialized: now 1786515203975490 us; error 36 us; skew 500 ppm
06:13:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.976331  8889 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:13:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.982805  8889 webserver.cc:533] Webserver started at http://127.6.39.129:45273/ using document root <none> and password file <none>
06:13:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.983661  8889 fs_manager.cc:362] Metadata directory not provided
06:13:23.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.983891  8889 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:13:23.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.992354  8889 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.001s
06:13:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.997025  8915 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:13:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.998134  8889 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
06:13:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:23.998481  8889 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:13:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:38 on dist-test-slave-vq2q"
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.000432  8889 fs_report.cc:389] FS layout report
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/wal
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/data
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:13:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:13:24.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.063808  8889 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:13:24.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.065608  8889 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:13:24.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.066144  8889 kserver.cc:163] Server-wide thread pool size limit: 3276
06:13:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.069067  8889 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:13:24.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.075730  8923 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:13:24.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.084398  8889 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:13:24.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.084606  8889 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
06:13:24.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.084820  8889 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:13:24.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.089646  8889 ts_tablet_manager.cc:616] Registered 1 tablets
06:13:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.089900  8889 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
06:13:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.090309  8923 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap starting.
06:13:24.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.178903  8923 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Log is configured to *not* fsync() on all Append() calls
06:13:24.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.279064  8923 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:13:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.279918  8923 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Bootstrap complete.
06:13:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.281818  8923 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent bootstrapping tablet: real 0.192s	user 0.137s	sys 0.049s
06:13:24.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.302824  8889 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.129:41443
06:13:24.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.303012  9049 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.129:41443 every 8 connection(s)
06:13:24.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.301236  8923 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:24.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.304368  8923 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Initialized, Role: FOLLOWER
06:13:24.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.305431  8923 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:24.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.306313  8889 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-0/data/info.pb
06:13:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.308743  8923 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4: Time spent starting tablet: real 0.027s	user 0.017s	sys 0.008s
06:13:24.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.315564  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 8889
06:13:24.316 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:24.346 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:24.347 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.130:42467
06:13:24.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:13:24.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.348466  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 8408
06:13:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.373162  9052 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:13:24.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.373759  9052 heartbeater.cc:461] Registering TS with master...
06:13:24.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.375043  9052 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:13:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.380367  9053 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:13:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.380424  6533 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.380681  9053 heartbeater.cc:461] Registering TS with master...
06:13:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.381650  9053 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:13:24.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.382922  6533 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.129:59077
06:13:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.385413  6446 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.386152  9054 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:13:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.386447  9054 heartbeater.cc:461] Registering TS with master...
06:13:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.387367  9054 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:24.390391  6354 ts_manager.cc:194] Re-registered known tserver with Master: 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:24.398 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] handling channelInactive
06:13:24.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state READY due to: connection closed
06:13:24.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:24.399 [DEBUG - Time-limited test] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:24.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.401 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.419 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:24.419 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:24.420 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1736099382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5040a7a9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1895aa51 -> wakeup thread Time-limited test)) after lookup
06:13:24.420 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:24.421 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.421 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.421 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.439 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:24.440 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:24.440 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [22ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [22ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1736099382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5040a7a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1895aa51 -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581))) after lookup
06:13:24.440 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:24.441 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.441 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.442 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.479 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:24.479 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:24.480 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [22ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [22ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [41ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [42ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [43ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [80ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1736099382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5040a7a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1895aa51 -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498))) after lookup
06:13:24.480 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:24.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.482 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.498 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:24.499 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:24.499 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=100), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [22ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [22ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [41ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [42ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [43ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [80ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [81ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [83ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [83ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [99ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1736099382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5040a7a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1895aa51 -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217))) after lookup
06:13:24.499 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:24.500 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.500 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:24.501 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:25.020 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:25.020 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:25.022 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=623), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [22ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [22ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [41ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [42ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [43ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [80ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [81ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [83ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [83ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [99ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [100ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [102ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [102ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [616ms] refreshing cache from master, [616ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [620ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1736099382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5040a7a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217) -> (continuation of Deferred@1711036284 after retry RPC@1946191249), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1895aa51 -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217) -> (continuation of Deferred@1711036284 after retry RPC@1946191249))) after lookup
06:13:25.023 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:25.024 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:25.025 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:25.025 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.389875  9052 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:13:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.444816  9069 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:13:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.445307  9069 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:25.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.447752  9069 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:25.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:25.452469  8920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:25.456477  8920 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.461712  8750 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "e93dadc87c1d4f888622d646f01f4da6" is_pre_election: true
06:13:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.463361  8920 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4; no voters: 7a785279bacd40deb619d9f334289da6, e93dadc87c1d4f888622d646f01f4da6
06:13:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.463989  9069 raft_consensus.cc:2749] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
06:13:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.628650  9075 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 7a785279bacd40deb619d9f334289da6)
06:13:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.629302  9075 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.631119  9075 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:25.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:25.634172  8673 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:25.638571  8673 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:25.640 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:25.640 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)[L],e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:25.641 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1241), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [22ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [22ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [41ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [42ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [43ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [80ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [81ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [83ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [83ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [99ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [100ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [102ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [102ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [616ms] refreshing cache from master, [616ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [620ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [624ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [626ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [626ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1236ms] refreshing cache from master, [1237ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1241ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1736099382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5040a7a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217) -> (continuation of Deferred@1711036284 after retry RPC@1946191249) -> (continuation of Deferred@760108875 after retry RPC@1569878910), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1895aa51 -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217) -> (continuation of Deferred@1711036284 after retry RPC@1946191249) -> (continuation of Deferred@760108875 after retry RPC@1569878910))) after lookup
06:13:25.642 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] connecting to peer
06:13:25.643 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:25.643 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:25.643 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467
06:13:25.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.645206  8995 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4" is_pre_election: true
06:13:25.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.645948  8995 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 8.
06:13:25.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.647272  8670 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, e93dadc87c1d4f888622d646f01f4da6; no voters: 7a785279bacd40deb619d9f334289da6
06:13:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.648020  9075 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 9 FOLLOWER]: Leader pre-election won for term 10
06:13:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.648294  9075 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 7a785279bacd40deb619d9f334289da6)
06:13:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.648619  9075 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 9 FOLLOWER]: Advancing to term 10
06:13:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.653875  9075 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.655431  9075 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 10 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129:41443)
06:13:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.656567  8995 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "e93dadc87c1d4f888622d646f01f4da6" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "08e4006c4cde42189b7529a3c7455cc4"
06:13:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.657075  8995 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 8 FOLLOWER]: Advancing to term 10
06:13:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:25.659094  8673 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111)
06:13:25.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.664477  8995 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e93dadc87c1d4f888622d646f01f4da6 in term 10.
06:13:25.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.665489  8670 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, e93dadc87c1d4f888622d646f01f4da6; no voters: 7a785279bacd40deb619d9f334289da6
06:13:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.666091  9075 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 10 FOLLOWER]: Leader election won for term 10
06:13:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.667695  9075 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 10 LEADER]: Becoming Leader. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Running, Role: LEADER
06:13:25.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.668711  9075 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.680541  6533 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 reported cstate change: term changed from 9 to 10, leader changed from 7a785279bacd40deb619d9f334289da6 (127.6.39.130) to e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131). New cstate: current_term: 10 leader_uuid: "e93dadc87c1d4f888622d646f01f4da6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: UNKNOWN } } }
06:13:25.939 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:25.939 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:25.940 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1540), Traces: [0ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [2ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [20ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [21ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [22ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [22ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [40ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [41ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [42ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [43ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [80ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [81ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [83ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [83ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [99ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [100ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [102ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [102ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [616ms] refreshing cache from master, [616ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [620ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [624ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [626ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [626ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1236ms] refreshing cache from master, [1237ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1241ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [1242ms] sending RPC to server 7a785279bacd40deb619d9f334289da6, [1244ms] received response from server 7a785279bacd40deb619d9f334289da6: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1244ms] delaying RPC due to: Network error: Failed to connect to peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467): Connection refused: /127.6.39.130:42467, [1536ms] refreshing cache from master, [1536ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1540ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1736099382(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@5040a7a9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217) -> (continuation of Deferred@1711036284 after retry RPC@1946191249) -> (continuation of Deferred@760108875 after retry RPC@1569878910) -> (continuation of Deferred@1917614091 after retry RPC@1007414463), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1895aa51 -> wakeup thread Time-limited test -> (continuation of Deferred@1389943087 after retry RPC@1368445581) -> (continuation of Deferred@1743404616 after retry RPC@817466498) -> (continuation of Deferred@1486348669 after retry RPC@615951217) -> (continuation of Deferred@1711036284 after retry RPC@1946191249) -> (continuation of Deferred@760108875 after retry RPC@1569878910) -> (continuation of Deferred@1917614091 after retry RPC@1007414463))) after lookup
06:13:25.940 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:25.942 [DEBUG - kudu-nio-0] (Connection.java:757) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] Successfully connected to peer
06:13:25.957 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x86885673, L:/127.0.0.1:33030 - R:/127.6.39.131:35835] using TOKEN/null
06:13:25.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.968118  8995 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 10 FOLLOWER]: Refusing update from remote peer e93dadc87c1d4f888622d646f01f4da6: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
06:13:25.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.969861  9081 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.001s
06:13:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:25.970680  8673 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 -> Peer 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467): Couldn't send request to peer 7a785279bacd40deb619d9f334289da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.130:42467: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:13:25.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:25.990629  9094 mvcc.cc:204] Tried to move back new op lower bound from 7317566283629387776 to 7317566282427387904. Current Snapshot: MvccSnapshot[applied={T|T < 7317566283629387776}]
06:13:26.035 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.130:42467
06:13:26.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:26.036685  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:42467
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42227
06:13:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:13:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:13:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:26.362432  9106 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:13:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:26.363039  9106 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:13:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:26.363682  9106 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:13:26.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:26.396528  9106 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:13:26.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:26.397554  9106 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.130
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:26.434630  9106 tablet_server_runner.cc:78] Tablet server non-default flags:
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.130:42467
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.130
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42227
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.130
06:13:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/logs
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:13:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:26.436369  9106 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:13:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:26.438263  9106 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:13:26.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:26.451169  9113 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:26.452371  9114 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:27.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:27.613273  9117 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:27.615769  9115 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1158 milliseconds
06:13:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.615869  9106 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:13:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.617219  9106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:13:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.619493  9106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:13:27.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.620853  9106 hybrid_clock.cc:648] HybridClock initialized: now 1786515207620809 us; error 42 us; skew 500 ppm
06:13:27.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.621721  9106 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:13:27.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.628060  9106 webserver.cc:533] Webserver started at http://127.6.39.130:42227/ using document root <none> and password file <none>
06:13:27.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.628998  9106 fs_manager.cc:362] Metadata directory not provided
06:13:27.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.629247  9106 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:13:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.638125  9106 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.003s	sys 0.005s
06:13:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.642982  9132 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:13:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.644186  9106 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
06:13:27.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.644567  9106 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:27.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a785279bacd40deb619d9f334289da6"
06:13:27.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:40 on dist-test-slave-vq2q"
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.646656  9106 fs_report.cc:389] FS layout report
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/wal
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/data
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:13:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:13:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.696108  9106 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:13:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.697724  9106 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:13:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.698244  9106 kserver.cc:163] Server-wide thread pool size limit: 3276
06:13:27.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.701203  9106 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:13:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.707207  9140 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:13:27.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.715858  9106 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:13:27.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.716097  9106 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
06:13:27.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.716418  9106 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:13:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.721088  9106 ts_tablet_manager.cc:616] Registered 1 tablets
06:13:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.721323  9106 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.001s	sys 0.005s
06:13:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.721738  9140 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap starting.
06:13:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.806349  9140 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Log is configured to *not* fsync() on all Append() calls
06:13:27.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.918401  9140 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:13:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.919401  9140 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Bootstrap complete.
06:13:27.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.921348  9140 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent bootstrapping tablet: real 0.200s	user 0.148s	sys 0.046s
06:13:27.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.943619  9140 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:27.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.947309  9140 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7a785279bacd40deb619d9f334289da6, State: Initialized, Role: FOLLOWER
06:13:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.948424  9140 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.952143  9140 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6: Time spent starting tablet: real 0.031s	user 0.029s	sys 0.000s
06:13:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.956967  9106 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.130:42467
06:13:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.957818  9268 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.130:42467 every 8 connection(s)
06:13:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.959702  9106 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-1/data/info.pb
06:13:27.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.961233  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 9106
06:13:27.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:27.990 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:27.991 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.6.39.131:35835
06:13:27.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:13:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:27.992240  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 8638
06:13:28.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.019560  9269 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:13:28.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.020040  9269 heartbeater.cc:461] Registering TS with master...
06:13:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.021514  9269 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:13:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.030017  6535 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:13:28.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.034830  6535 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.130:38403
06:13:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.037035  9270 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:13:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.037420  9270 heartbeater.cc:461] Registering TS with master...
06:13:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.038277  9270 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:13:28.041 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:28.042130  9274 negotiation.cc:336] Failed RPC negotiation. Trace:
06:13:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:27.972684 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.6.39.131:54297 (local address 127.6.39.130:42467)
06:13:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:27.996842 (+ 24158us) server_negotiation.cc:207] Beginning negotiation
06:13:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:27.996878 (+    36us) server_negotiation.cc:400] Waiting for connection header
06:13:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:27.997010 (+   132us) server_negotiation.cc:408] Connection header received
06:13:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:27.997787 (+   777us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
06:13:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:27.997878 (+    91us) server_negotiation.cc:462] Received NEGOTIATE request from client
06:13:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:27.998485 (+   607us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
06:13:28.044 [DEBUG - kudu-nio-0] (Connection.java:239) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] handling channelInactive
06:13:28.044 [DEBUG - kudu-nio-0] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state READY due to: connection closed
06:13:28.045 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: connection closed
06:13:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:28.002498 (+  4013us) server_negotiation.cc:227] Negotiated authn=SASL
06:13:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0812 06:13:28.040263 (+ 37765us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.6.39.131:54297: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:13:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":22605,"thread_start_us":24666,"threads_started":1}
06:13:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.042833  6446 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:13:28.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.050998  9273 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:13:28.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.051294  9273 heartbeater.cc:461] Registering TS with master...
06:13:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.051803  9273 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:13:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:28.054616  6354 ts_manager.cc:194] Re-registered known tserver with Master: 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467)
06:13:28.058 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:28.058 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:28.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test)) after lookup
06:13:28.059 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:28.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.060 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.060 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.079 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:28.079 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:28.079 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [18ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [19ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [19ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [37ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723))) after lookup
06:13:28.080 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:28.081 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.081 [DEBUG - kudu-nio-2] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.081 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.119 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:28.119 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:28.119 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [18ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [19ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [19ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [37ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [38ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [40ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [40ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161))) after lookup
06:13:28.120 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:28.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.121 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.219 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:28.220 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:28.220 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=179), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [18ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [19ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [19ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [37ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [38ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [40ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [40ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [79ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [80ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [80ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [178ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307))) after lookup
06:13:28.221 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:28.222 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.222 [DEBUG - kudu-nio-4] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.222 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.539 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:28.539 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:28.539 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=498), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [18ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [19ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [19ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [37ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [38ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [40ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [40ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [79ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [80ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [80ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [178ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [179ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [181ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [181ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [494ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [497ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673))) after lookup
06:13:28.540 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:28.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.541 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.960 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:28.960 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:28.961 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=919), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [18ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [19ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [19ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [37ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [38ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [40ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [40ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [79ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [80ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [80ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [178ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [179ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [181ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [181ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [494ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [497ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [499ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [500ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [500ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [914ms] refreshing cache from master, [914ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [919ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673) -> (continuation of Deferred@1630098427 after retry RPC@16035835), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673) -> (continuation of Deferred@1630098427 after retry RPC@16035835))) after lookup
06:13:28.961 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:28.962 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.962 [DEBUG - kudu-nio-6] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:28.963 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.047524  9269 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:13:29.059 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:29.059 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443),7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)[L]]}]
06:13:29.060 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1018), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [18ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [19ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [19ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [37ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [38ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [40ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [40ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [79ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [80ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [80ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [178ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [179ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [181ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [181ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [494ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [497ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [499ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [500ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [500ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [914ms] refreshing cache from master, [914ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [919ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [920ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [921ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [922ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [1014ms] refreshing cache from master, [1014ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1018ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673) -> (continuation of Deferred@1630098427 after retry RPC@16035835) -> (continuation of Deferred@1916781246 after retry RPC@1469132917), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673) -> (continuation of Deferred@1630098427 after retry RPC@16035835) -> (continuation of Deferred@1916781246 after retry RPC@1469132917))) after lookup
06:13:29.060 [DEBUG - kudu-nio-2] (Connection.java:748) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] connecting to peer
06:13:29.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:29.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] cleaning up while in state CONNECTING due to: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:29.062 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835) for tablet b27f111dffc74b94ba5a7a2a6d8af055: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835
06:13:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.348104  9287 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:13:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.348512  9287 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:29.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.350135  9287 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:29.356959  8920 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:13:29.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.361896  9215 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6" is_pre_election: true
06:13:29.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.362536  9215 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 9.
06:13:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.363677  8920 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:13:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.364274  9287 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 10 FOLLOWER]: Leader pre-election won for term 11
06:13:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.364554  9287 raft_consensus.cc:493] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 10 FOLLOWER]: Starting leader election (detected failure of leader e93dadc87c1d4f888622d646f01f4da6)
06:13:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.364773  9287 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 10 FOLLOWER]: Advancing to term 11
06:13:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.368955  9287 raft_consensus.cc:515] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.370484  9287 leader_election.cc:290] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 11 election: Requested vote from peers 7a785279bacd40deb619d9f334289da6 (127.6.39.130:42467), e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.371129  9215 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "b27f111dffc74b94ba5a7a2a6d8af055" candidate_uuid: "08e4006c4cde42189b7529a3c7455cc4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "7a785279bacd40deb619d9f334289da6"
06:13:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.371708  9215 raft_consensus.cc:3060] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 9 FOLLOWER]: Advancing to term 11
06:13:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:29.374249  8920 leader_election.cc:336] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111)
06:13:29.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.378521  9215 raft_consensus.cc:2468] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 08e4006c4cde42189b7529a3c7455cc4 in term 11.
06:13:29.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.379382  8920 leader_election.cc:304] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 08e4006c4cde42189b7529a3c7455cc4, 7a785279bacd40deb619d9f334289da6; no voters: e93dadc87c1d4f888622d646f01f4da6
06:13:29.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.379973  9287 raft_consensus.cc:2804] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 11 FOLLOWER]: Leader election won for term 11
06:13:29.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.381001  9287 raft_consensus.cc:697] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [term 11 LEADER]: Becoming Leader. State: Replica: 08e4006c4cde42189b7529a3c7455cc4, State: Running, Role: LEADER
06:13:29.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.381954  9287 consensus_queue.cc:237] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:29.391119  6535 catalog_manager.cc:5739] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 reported cstate change: term changed from 10 to 11, leader changed from e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131) to 08e4006c4cde42189b7529a3c7455cc4 (127.6.39.129). New cstate: current_term: 11 leader_uuid: "08e4006c4cde42189b7529a3c7455cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } health_report { overall_health: HEALTHY } } }
06:13:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:29.922446  8920 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 -> Peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Couldn't send request to peer e93dadc87c1d4f888622d646f01f4da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:13:30.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:30.003475  9215 raft_consensus.cc:1275] T b27f111dffc74b94ba5a7a2a6d8af055 P 7a785279bacd40deb619d9f334289da6 [term 11 FOLLOWER]: Refusing update from remote peer 08e4006c4cde42189b7529a3c7455cc4: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
06:13:30.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:30.005064  9311 consensus_queue.cc:1048] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
06:13:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:30.851030  8920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111) [suppressed 4 similar messages]
06:13:31.659 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet b27f111dffc74b94ba5a7a2a6d8af055 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217' with partition [<start>, <end>)
06:13:31.659 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=b27f111dffc74b94ba5a7a2a6d8af055@[08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)[L],7a785279bacd40deb619d9f334289da6(127.6.39.130:42467),e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)]}]
06:13:31.660 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=b27f111dffc74b94ba5a7a2a6d8af055, attempt=16, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=3618), Traces: [0ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [4ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [14ms] refreshing cache from master, [14ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [17ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [18ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [19ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [19ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [37ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [38ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [40ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [40ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [77ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [79ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [80ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [80ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [174ms] refreshing cache from master, [174ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [178ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [179ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [181ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [181ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [494ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [497ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [499ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [500ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [500ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [914ms] refreshing cache from master, [914ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [919ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [920ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [921ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [922ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [1014ms] refreshing cache from master, [1014ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [1018ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, [1019ms] sending RPC to server e93dadc87c1d4f888622d646f01f4da6, [1021ms] received response from server e93dadc87c1d4f888622d646f01f4da6: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [1021ms] delaying RPC due to: Network error: Failed to connect to peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835): Connection refused: /127.6.39.131:35835, [3614ms] refreshing cache from master, [3615ms] Sub RPC GetTableLocations: sending RPC to server master-127.6.39.188:40371, [3618ms] Sub RPC GetTableLocations: received response from server master-127.6.39.188:40371: OK, deferred=Deferred@1330788166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f33f036c7f8@2cdf4407 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673) -> (continuation of Deferred@1630098427 after retry RPC@16035835) -> (continuation of Deferred@1916781246 after retry RPC@1469132917) -> (continuation of Deferred@1877750013 after retry RPC@824227821), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@15d5f5a3 -> wakeup thread Time-limited test -> (continuation of Deferred@1067612897 after retry RPC@98629723) -> (continuation of Deferred@1244218184 after retry RPC@883642161) -> (continuation of Deferred@912880504 after retry RPC@1481994307) -> (continuation of Deferred@1312760325 after retry RPC@1025213673) -> (continuation of Deferred@1630098427 after retry RPC@16035835) -> (continuation of Deferred@1916781246 after retry RPC@1469132917) -> (continuation of Deferred@1877750013 after retry RPC@824227821))) after lookup
06:13:31.660 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] connecting to peer
06:13:31.661 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] Successfully connected to peer
06:13:31.678 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xe6c8a2db, L:/127.0.0.1:52938 - R:/127.6.39.129:41443] using TOKEN/null
06:13:31.724 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.6.39.131:35835
06:13:31.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:31.726156  6302 external_mini_cluster.cc:1373] Running /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:13:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:13:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.059322  9342 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
06:13:32.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.059819  9342 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:13:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.060282  9342 flags.cc:432] Enabled unsafe flag: --never_fsync=true
06:13:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.093683  9342 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
06:13:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.094549  9342 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.39.131
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:32.130561  9342 tablet_server_runner.cc:78] Tablet server non-default flags:
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.39.148:40429
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.39.131:35835
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.39.131
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.39.190:40895,127.6.39.189:43971,127.6.39.188:40371
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.39.131
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/logs
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
06:13:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:13:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
06:13:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5aabff03afe514f4e9286d8ef2d8bf6bed19fb06
06:13:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:13:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 05:53:07 UTC on e7f111948823
06:13:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17646
06:13:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:13:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:32.131963  9342 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:13:32.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:32.134135  9342 file_cache.cc:504] Constructed file cache file cache with capacity 419430
06:13:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.147882  9351 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.541026  8920 consensus_peers.cc:597] T b27f111dffc74b94ba5a7a2a6d8af055 P 08e4006c4cde42189b7529a3c7455cc4 -> Peer e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835): Couldn't send request to peer e93dadc87c1d4f888622d646f01f4da6. Status: Network error: Client connection negotiation failed: client connection to 127.6.39.131:35835: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:13:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:32.149292  9352 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:33.330466  9356 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:13:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:33.332847  9342 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.182s	user 0.000s	sys 0.007s
06:13:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 06:13:33.333163  9342 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.183s	user 0.000s	sys 0.007s
06:13:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.333408  9342 server_base.cc:1061] running on GCE node
06:13:33.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.334435  9342 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:13:33.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.336725  9342 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:13:33.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.338088  9342 hybrid_clock.cc:648] HybridClock initialized: now 1786515213338051 us; error 58 us; skew 500 ppm
06:13:33.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.339113  9342 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:13:33.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.346385  9342 webserver.cc:533] Webserver started at http://127.6.39.131:41543/ using document root <none> and password file <none>
06:13:33.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.347525  9342 fs_manager.cc:362] Metadata directory not provided
06:13:33.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.347738  9342 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:13:33.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.355955  9342 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
06:13:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.361191  9367 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
06:13:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.362586  9342 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
06:13:33.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.362910  9342 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data,/tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e93dadc87c1d4f888622d646f01f4da6"
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 06:12:42 on dist-test-slave-vq2q"
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.364900  9342 fs_report.cc:389] FS layout report
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/wal
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/data
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:13:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:13:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:13:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:13:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:13:33.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.432087  9342 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
06:13:33.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.433693  9342 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:13:33.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.434123  9342 kserver.cc:163] Server-wide thread pool size limit: 3276
06:13:33.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.437055  9342 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
06:13:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.443632  9374 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
06:13:33.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.451859  9342 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
06:13:33.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.452078  9342 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
06:13:33.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.452311  9342 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
06:13:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.456997  9342 ts_tablet_manager.cc:616] Registered 1 tablets
06:13:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.457269  9342 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
06:13:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.457713  9374 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap starting.
06:13:33.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.544078  9374 log.cc:826] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Log is configured to *not* fsync() on all Append() calls
06:13:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.646149  9342 rpc_server.cc:307] RPC server started. Bound to: 127.6.39.131:35835
06:13:33.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.646402  9503 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.6.39.131:35835 every 8 connection(s)
06:13:33.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.648903  9342 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0kB5X1/test-tmp/mini-kudu-cluster8352327500704487523/ts-2/data/info.pb
06:13:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.658973  6302 external_mini_cluster.cc:1435] Started /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu as pid 9342
06:13:33.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
06:13:33.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.664052  9374 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:13:33.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.666276  9374 tablet_bootstrap.cc:492] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Bootstrap complete.
06:13:33.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.668162  9374 ts_tablet_manager.cc:1403] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent bootstrapping tablet: real 0.211s	user 0.159s	sys 0.045s
06:13:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.695405  9374 raft_consensus.cc:359] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.699182  9374 raft_consensus.cc:740] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: e93dadc87c1d4f888622d646f01f4da6, State: Initialized, Role: FOLLOWER
06:13:33.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.700664  9374 consensus_queue.cc:260] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a785279bacd40deb619d9f334289da6" member_type: VOTER last_known_addr { host: "127.6.39.130" port: 42467 } } peers { permanent_uuid: "e93dadc87c1d4f888622d646f01f4da6" member_type: VOTER last_known_addr { host: "127.6.39.131" port: 35835 } } peers { permanent_uuid: "08e4006c4cde42189b7529a3c7455cc4" member_type: VOTER last_known_addr { host: "127.6.39.129" port: 41443 } }
06:13:33.719 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:33.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.754849  9504 heartbeater.cc:344] Connected to a master server at 127.6.39.188:40371
06:13:33.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.755542  9504 heartbeater.cc:461] Registering TS with master...
06:13:33.757 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786515163217, tablet=b27f111dffc74b94ba5a7a2a6d8af055, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
06:13:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.757921  9374 ts_tablet_manager.cc:1434] T b27f111dffc74b94ba5a7a2a6d8af055 P e93dadc87c1d4f888622d646f01f4da6: Time spent starting tablet: real 0.089s	user 0.049s	sys 0.028s
06:13:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.757982  9504 heartbeater.cc:507] Master 127.6.39.188:40371 requested a full tablet report, sending...
06:13:33.763 [DEBUG - main] (Connection.java:575) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] disconnecting while in state READY
06:13:33.765 [DEBUG - main] (Connection.java:575) [peer 7a785279bacd40deb619d9f334289da6(127.6.39.130:42467)] disconnecting while in state TERMINATED
06:13:33.765 [DEBUG - main] (Connection.java:575) [peer master-127.6.39.189:43971(127.6.39.189:43971)] disconnecting while in state READY
06:13:33.765 [DEBUG - main] (Connection.java:575) [peer e93dadc87c1d4f888622d646f01f4da6(127.6.39.131:35835)] disconnecting while in state TERMINATED
06:13:33.765 [DEBUG - main] (Connection.java:575) [peer master-127.6.39.188:40371(127.6.39.188:40371)] disconnecting while in state READY
06:13:33.766 [DEBUG - main] (Connection.java:575) [peer master-127.6.39.190:40895(127.6.39.190:40895)] disconnecting while in state READY
06:13:33.772 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] handling channelInactive
06:13:33.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 08e4006c4cde42189b7529a3c7455cc4(127.6.39.129:41443)] cleaning up while in state READY due to: connection closed
06:13:33.773 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.39.189:43971(127.6.39.189:43971)] handling channelInactive
06:13:33.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.39.189:43971(127.6.39.189:43971)] cleaning up while in state READY due to: connection closed
06:13:33.776 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.6.39.188:40371(127.6.39.188:40371)] handling channelInactive
06:13:33.777 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.6.39.188:40371(127.6.39.188:40371)] cleaning up while in state READY due to: connection closed
06:13:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.772429  9505 heartbeater.cc:344] Connected to a master server at 127.6.39.189:43971
06:13:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.772859  9505 heartbeater.cc:461] Registering TS with master...
06:13:33.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.773988  6535 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:33.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.779048  9505 heartbeater.cc:507] Master 127.6.39.189:43971 requested a full tablet report, sending...
06:13:33.783 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:13:33.784 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.6.39.190:40895(127.6.39.190:40895)] handling channelInactive
06:13:33.784 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.39.190:40895(127.6.39.190:40895)] cleaning up while in state READY due to: connection closed
06:13:33.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.785269  6446 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:33.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.787922  9507 heartbeater.cc:344] Connected to a master server at 127.6.39.190:40895
06:13:33.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.788321  9507 heartbeater.cc:461] Registering TS with master...
06:13:33.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.789894  9507 heartbeater.cc:507] Master 127.6.39.190:40895 requested a full tablet report, sending...
06:13:33.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.793381  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 8889
06:13:33.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.795531  6353 ts_manager.cc:194] Re-registered known tserver with Master: e93dadc87c1d4f888622d646f01f4da6 (127.6.39.131:35835)
06:13:33.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.800376  6535 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.6.39.131:40791
06:13:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.805671  9504 heartbeater.cc:499] Master 127.6.39.188:40371 was elected leader, sending a full tablet report...
06:13:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.829289  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 9106
06:13:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.862298  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 9342
06:13:33.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.902650  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 6315
06:13:33.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.937678  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 6402
06:13:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 06:13:33.970140  6302 external_mini_cluster.cc:1671] Killing /tmp/dist-test-task0kB5X1/build/tsan/bin/kudu with pid 6489
06:13:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T06:13:35Z chronyd exiting

Time: 68.111

OK (1 test)

