JUnit version 4.13.2
.02:03:47.232 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1779069827223
02:03:47.290 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
02:03:47.371 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:03:47.390 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskFcNZ7a/test-tmp
02:03:47.399 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:03:47.403 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskFcNZ7a/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:03:47.629 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

02:03:47.823 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
02:03:47.826 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

02:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:47.831037 13225 mini_oidc.cc:150] Starting JWKS server
02:03:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:47.834831 13225 webserver.cc:533] Webserver started at http://127.0.0.1:37529/ using document root <none> and password file <none>
02:03:47.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:47.835418 13225 mini_oidc.cc:186] Starting OIDC Discovery server
02:03:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:47.838043 13225 webserver.cc:533] Webserver started at http://127.0.0.1:42775/ using document root <none> and password file <none>
02:03:47.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T02:03:47Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:03:47.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T02:03:47Z Disabled control of system clock
02:03:47.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:47.904959 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/wal
02:03:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data
02:03:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/logs
02:03:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data/info.pb
02:03:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:47.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:47.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:47.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:47.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.126:34017
02:03:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.126
02:03:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:47.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:47.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:47.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:47.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:47.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:47.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37529/jwks.json with env {}
02:03:48.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.230954 13238 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.231544 13238 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.231980 13238 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.267098 13238 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.267402 13238 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.267598 13238 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.267940 13238 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.268275 13238 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.268533 13238 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37529/jwks.json
02:03:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:48.311305 13238 master_runner.cc:387] Master server non-default flags:
02:03:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data
02:03:48.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/wal
02:03:48.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:48.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:48.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:48.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.126:34017
02:03:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37529/jwks.json
02:03:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data/info.pb
02:03:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.126
02:03:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/logs
02:03:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:03:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:03:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:03:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:03:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:48.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:48.312712 13238 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:48.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:48.315518 13238 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:48.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.331174 13244 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:50.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:48.331291 13245 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:50.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:49.733134 13243 debug-util.cc:398] Leaking SignalData structure 0x7b08000271e0 after lost signal to thread 13238
02:03:50.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.200992 13238 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.871s	user 0.651s	sys 1.214s
02:03:50.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.203172 13238 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.874s	user 0.651s	sys 1.215s
02:03:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.203825 13247 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.203737 13238 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.204851 13246 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1872 milliseconds
02:03:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.209335 13238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.212421 13238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:50.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.213979 13238 hybrid_clock.cc:648] HybridClock initialized: now 1779069830213889 us; error 66 us; skew 500 ppm
02:03:50.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.215106 13238 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:50.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.222182 13238 webserver.cc:533] Webserver started at http://127.12.234.126:42591/ using document root <none> and password file <none>
02:03:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.223295 13238 fs_manager.cc:362] Metadata directory not provided
02:03:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.223512 13238 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:50.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.223908 13238 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.228557 13238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data/instance:
02:03:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d29d08e92eda49708a109dddae4c40b3"
02:03:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:50 on dist-test-slave-9xtj"
02:03:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.229923 13238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/wal/instance:
02:03:50.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d29d08e92eda49708a109dddae4c40b3"
02:03:50.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:50 on dist-test-slave-9xtj"
02:03:50.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.239993 13238 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.009s	sys 0.001s
02:03:50.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.246388 13254 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:03:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.247859 13238 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.004s
02:03:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.248221 13238 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/wal
02:03:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d29d08e92eda49708a109dddae4c40b3"
02:03:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:50 on dist-test-slave-9xtj"
02:03:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.248724 13238 fs_report.cc:389] FS layout report
02:03:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/wal
02:03:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/wal
02:03:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data/data
02:03:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.303644 13238 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:03:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.305471 13238 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.305895 13238 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.387743 13238 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.126:34017
02:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.387832 13305 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.126:34017 every 8 connection(s)
02:03:50.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.390640 13238 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/data/info.pb
02:03:50.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.394392 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 13238
02:03:50.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.394971 13225 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-0/wal/instance
02:03:50.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.398593 13306 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.400710 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/wal
02:03:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data
02:03:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/logs
02:03:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data/info.pb
02:03:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.125:39317
02:03:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.125
02:03:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37529/jwks.json with env {}
02:03:50.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.416006 13306 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } has no permanent_uuid. Determining permanent_uuid...
02:03:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.452011 13306 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } has no permanent_uuid. Determining permanent_uuid...
02:03:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.455083 13257 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:50.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.459842 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.125:39317: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.509518 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } attempt: 1
02:03:50.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.514595 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.125:39317: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.583241 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } attempt: 2
02:03:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.587960 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.125:39317: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:50.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.679522 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } attempt: 3
02:03:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.684516 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.125:39317: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.764598 13308 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.765169 13308 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.765618 13308 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.800964 13308 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.801285 13308 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.801479 13308 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.801661 13308 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.801841 13308 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.801978 13308 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37529/jwks.json
02:03:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.828042 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } attempt: 4
02:03:50.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.832208 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.125:39317: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:50.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.843915 13308 master_runner.cc:387] Master server non-default flags:
02:03:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data
02:03:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/wal
02:03:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.125:39317
02:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37529/jwks.json
02:03:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data/info.pb
02:03:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.125
02:03:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:50.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:50.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:50.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:50.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:50.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:50.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/logs
02:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:03:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:03:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:03:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.845456 13308 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:50.847275 13308 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:50.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.863511 13317 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:51.131676 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } attempt: 5
02:03:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:51.139142 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.125:39317: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:51.686810 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } attempt: 6
02:03:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:51.709643 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.125:39317: Network error: Client connection negotiation failed: client connection to 127.12.234.125:39317: connect: Connection refused (error 111)
02:03:52.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:50.863632 13318 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.153926 13320 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:52.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.157560 13319 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1286 milliseconds
02:03:52.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.157752 13308 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.159386 13308 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:52.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.164463 13308 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:52.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.166771 13308 hybrid_clock.cc:648] HybridClock initialized: now 1779069832166690 us; error 95 us; skew 500 ppm
02:03:52.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.167884 13308 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:52.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.174943 13308 webserver.cc:533] Webserver started at http://127.12.234.125:44407/ using document root <none> and password file <none>
02:03:52.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.176227 13308 fs_manager.cc:362] Metadata directory not provided
02:03:52.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.176498 13308 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:52.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.177139 13308 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.182055 13308 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data/instance:
02:03:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc469968c1bd4b9bb216a6805aafd0fd"
02:03:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:52 on dist-test-slave-9xtj"
02:03:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.183748 13308 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/wal/instance:
02:03:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc469968c1bd4b9bb216a6805aafd0fd"
02:03:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:52 on dist-test-slave-9xtj"
02:03:52.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.192454 13308 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.007s	sys 0.000s
02:03:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.199151 13329 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:03:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.200706 13308 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.007s	sys 0.000s
02:03:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.201148 13308 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/wal
02:03:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc469968c1bd4b9bb216a6805aafd0fd"
02:03:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:52 on dist-test-slave-9xtj"
02:03:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.201683 13308 fs_report.cc:389] FS layout report
02:03:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/wal
02:03:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/wal
02:03:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data/data
02:03:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:52.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:52.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.270289 13308 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:03:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.272104 13308 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.272696 13308 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.351603 13308 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.125:39317
02:03:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.351696 13380 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.125:39317 every 8 connection(s)
02:03:52.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.355387 13308 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/data/info.pb
02:03:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.359804 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 13308
02:03:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.360352 13225 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-1/wal/instance
02:03:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.363480 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/wal
02:03:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data
02:03:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/logs
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data/info.pb
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.124:38173
02:03:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.124
02:03:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37529/jwks.json with env {}
02:03:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.363911 13381 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:52.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.380541 13381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } has no permanent_uuid. Determining permanent_uuid...
02:03:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.416280 13381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } has no permanent_uuid. Determining permanent_uuid...
02:03:52.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.432325 13381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } has no permanent_uuid. Determining permanent_uuid...
02:03:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.435216 13330 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.439738 13381 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.489288 13381 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 1
02:03:52.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.493552 13381 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.562151 13381 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 2
02:03:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.566959 13381 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.658461 13381 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 3
02:03:52.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.663117 13381 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.710660 13383 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:52.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.711239 13383 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.711779 13383 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.746085 13383 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.746415 13383 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.746619 13383 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.746805 13383 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.746984 13383 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.747123 13383 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37529/jwks.json
02:03:52.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.770398 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } attempt: 7
02:03:52.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.784263 13306 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } has no permanent_uuid. Determining permanent_uuid...
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.790684 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.791059 13383 master_runner.cc:387] Master server non-default flags:
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data
02:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/wal
02:03:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.124:38173
02:03:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37529/jwks.json
02:03:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data/info.pb
02:03:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.124
02:03:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/logs
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:03:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:03:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:03:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.792457 13383 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.794158 13383 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:52.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.806490 13395 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:52.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.806629 13381 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 4
02:03:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.813064 13381 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.849231 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 1
02:03:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.856665 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:52.938462 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 2
02:03:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.946242 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:53.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:53.031875 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 3
02:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:53.039400 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:53.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:53.112668 13381 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 5
02:03:53.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:53.120193 13381 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:53.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:53.180084 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 4
02:03:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:53.187930 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:53.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:53.471647 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 5
02:03:53.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:53.480115 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:53.667956 13381 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 6
02:03:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:53.681113 13381 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:54.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.032780 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 6
02:03:54.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.043249 13306 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.234.124:38173: Network error: Client connection negotiation failed: client connection to 127.12.234.124:38173: connect: Connection refused (error 111)
02:03:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:52.807106 13396 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.092053 13398 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:54.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.097446 13383 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.291s	user 0.469s	sys 0.821s
02:03:54.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.097771 13383 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.291s	user 0.469s	sys 0.821s
02:03:54.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.098440 13383 server_base.cc:1061] running on GCE node
02:03:54.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.099694 13383 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:54.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.103255 13383 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:54.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.104739 13383 hybrid_clock.cc:648] HybridClock initialized: now 1779069834104685 us; error 69 us; skew 500 ppm
02:03:54.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.106006 13383 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:54.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.117921 13383 webserver.cc:533] Webserver started at http://127.12.234.124:46353/ using document root <none> and password file <none>
02:03:54.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.119410 13383 fs_manager.cc:362] Metadata directory not provided
02:03:54.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.119649 13383 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:54.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.120127 13383 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.125205 13383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data/instance:
02:03:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f217dc377e84bf7abaadf66f72b234d"
02:03:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:54 on dist-test-slave-9xtj"
02:03:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.126406 13383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/wal/instance:
02:03:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f217dc377e84bf7abaadf66f72b234d"
02:03:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:54 on dist-test-slave-9xtj"
02:03:54.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.135955 13383 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.006s	sys 0.004s
02:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.144064 13409 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:03:54.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.145689 13383 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.001s	sys 0.002s
02:03:54.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.146139 13383 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/wal
02:03:54.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f217dc377e84bf7abaadf66f72b234d"
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:54 on dist-test-slave-9xtj"
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.146644 13383 fs_report.cc:389] FS layout report
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/wal
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/wal
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data/data
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:54.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.222987 13383 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:03:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.224876 13383 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.225455 13383 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.318609 13383 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.124:38173
02:03:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.318677 13460 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.124:38173 every 8 connection(s)
02:03:54.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.322736 13383 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/data/info.pb
02:03:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.328408 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 13383
02:03:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.328914 13225 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/master-2/wal/instance
02:03:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.330879 13461 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:54.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.355015 13461 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } has no permanent_uuid. Determining permanent_uuid...
02:03:54.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.397490 13461 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } has no permanent_uuid. Determining permanent_uuid...
02:03:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.412689 13461 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } has no permanent_uuid. Determining permanent_uuid...
02:03:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.431536 13461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d: Bootstrap starting.
02:03:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.439649 13461 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d: Neither blocks nor log segments found. Creating new log.
02:03:54.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.442440 13461 log.cc:826] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d: Log is configured to *not* fsync() on all Append() calls
02:03:54.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.448911 13461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d: No bootstrap required, opened a new log
02:03:54.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.472654 13461 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:54.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.473438 13461 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:54.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.473748 13461 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f217dc377e84bf7abaadf66f72b234d, State: Initialized, Role: FOLLOWER
02:03:54.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.474756 13461 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.477545 13470 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.478870 13470 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.479768 13461 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: configured and running, proceeding with master startup.
02:03:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.482080 13415 tablet.cc:2404] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:03:54.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.494354 13470 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:54.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.495054 13470 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.499514 13470 leader_election.cc:290] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29d08e92eda49708a109dddae4c40b3 (127.12.234.126:34017), dc469968c1bd4b9bb216a6805aafd0fd (127.12.234.125:39317)
02:03:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.528446 13481 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.528837 13481 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:54.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.514737 13383 master_runner.cc:187] Error getting master registration for 127.12.234.126:34017: OK, instance_id { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" instance_seqno: 1779069830363833 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:03:54.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.550956 13383 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:03:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.741743 13381 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 7
02:03:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.769466 13381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: Bootstrap starting.
02:03:54.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.778501 13381 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: Neither blocks nor log segments found. Creating new log.
02:03:54.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.784401 13381 log.cc:826] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: Log is configured to *not* fsync() on all Append() calls
02:03:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.790803 13381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: No bootstrap required, opened a new log
02:03:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.812844 13381 raft_consensus.cc:359] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.814435 13381 raft_consensus.cc:385] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:54.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.815192 13381 raft_consensus.cc:740] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc469968c1bd4b9bb216a6805aafd0fd, State: Initialized, Role: FOLLOWER
02:03:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.816776 13381 consensus_queue.cc:260] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.820227 13485 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.821570 13485 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.822345 13381 sys_catalog.cc:565] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [sys.catalog]: configured and running, proceeding with master startup.
02:03:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.822567 13356 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f217dc377e84bf7abaadf66f72b234d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" is_pre_election: true
02:03:54.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.823489 13356 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f217dc377e84bf7abaadf66f72b234d in term 0.
02:03:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.825750 13412 leader_election.cc:304] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f217dc377e84bf7abaadf66f72b234d, dc469968c1bd4b9bb216a6805aafd0fd; no voters: 
02:03:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.827800 13470 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Leader pre-election won for term 1
02:03:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.828238 13470 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:03:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.828606 13470 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 0 FOLLOWER]: Advancing to term 1
02:03:54.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.839356 13470 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:54.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.854913 13470 leader_election.cc:290] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [CANDIDATE]: Term 1 election: Requested vote from peers d29d08e92eda49708a109dddae4c40b3 (127.12.234.126:34017), dc469968c1bd4b9bb216a6805aafd0fd (127.12.234.125:39317)
02:03:54.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.857689 13356 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f217dc377e84bf7abaadf66f72b234d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc469968c1bd4b9bb216a6805aafd0fd"
02:03:54.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.858455 13356 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [term 0 FOLLOWER]: Advancing to term 1
02:03:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.879823 13356 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f217dc377e84bf7abaadf66f72b234d in term 1.
02:03:54.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.884997 13412 leader_election.cc:304] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f217dc377e84bf7abaadf66f72b234d, dc469968c1bd4b9bb216a6805aafd0fd; no voters: 
02:03:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.887172 13470 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 1 FOLLOWER]: Leader election won for term 1
02:03:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.894136 13470 raft_consensus.cc:697] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [term 1 LEADER]: Becoming Leader. State: Replica: 0f217dc377e84bf7abaadf66f72b234d, State: Running, Role: LEADER
02:03:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.895181 13470 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.915709 13497 catalog_manager.cc:1593] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:54.920948 13497 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.909020 13496 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f217dc377e84bf7abaadf66f72b234d. Latest consensus state: current_term: 1 leader_uuid: "0f217dc377e84bf7abaadf66f72b234d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.921653 13496 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: This master's current role is: LEADER
02:03:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.930123 13500 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
02:03:54.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.895552 13308 master_runner.cc:187] Error getting master registration for 127.12.234.126:34017: OK, instance_id { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" instance_seqno: 1779069830363833 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:03:54.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.945303 13308 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:03:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.947770 13500 catalog_manager.cc:1519] Initializing Kudu cluster ID...
02:03:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.967561 13356 raft_consensus.cc:1275] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [term 1 FOLLOWER]: Refusing update from remote peer 0f217dc377e84bf7abaadf66f72b234d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:03:54.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:54.969681 13470 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:03:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.005113 13485 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f217dc377e84bf7abaadf66f72b234d. Latest consensus state: current_term: 1 leader_uuid: "0f217dc377e84bf7abaadf66f72b234d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.005852 13485 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [sys.catalog]: This master's current role is: FOLLOWER
02:03:55.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.005880 13496 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0f217dc377e84bf7abaadf66f72b234d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:55.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.006664 13496 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: This master's current role is: LEADER
02:03:55.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.011978 13500 catalog_manager.cc:1382] Generated new cluster ID: 9670e36df2ca475eb34d037eef54deee
02:03:55.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.012331 13500 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
02:03:55.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.030938 13485 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0f217dc377e84bf7abaadf66f72b234d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:55.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.031672 13485 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd [sys.catalog]: This master's current role is: FOLLOWER
02:03:55.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.076745 13306 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } attempt: 7
02:03:55.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.100081 13500 catalog_manager.cc:1405] Generated new certificate authority record
02:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.102306 13500 catalog_manager.cc:1539] Loading token signing keys...
02:03:55.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.128954 13306 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3: Bootstrap starting.
02:03:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.135784 13306 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3: Neither blocks nor log segments found. Creating new log.
02:03:55.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.147141 13306 log.cc:826] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3: Log is configured to *not* fsync() on all Append() calls
02:03:55.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.145701 13500 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d: Generated new TSK 0
02:03:55.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.154762 13500 catalog_manager.cc:1549] Initializing in-progress tserver states...
02:03:55.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.161087 13306 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3: No bootstrap required, opened a new log
02:03:55.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.198161 13306 raft_consensus.cc:359] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:55.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.199067 13306 raft_consensus.cc:385] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:55.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.199401 13306 raft_consensus.cc:740] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d29d08e92eda49708a109dddae4c40b3, State: Initialized, Role: FOLLOWER
02:03:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.200783 13306 consensus_queue.cc:260] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } }
02:03:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.206660 13279 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [term 0 FOLLOWER]: Advancing to term 1
02:03:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.214875 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:03:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:55.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:55.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:03:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:0
02:03:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.206880 13280 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f217dc377e84bf7abaadf66f72b234d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d29d08e92eda49708a109dddae4c40b3"
02:03:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.207072 13281 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f217dc377e84bf7abaadf66f72b234d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d29d08e92eda49708a109dddae4c40b3" is_pre_election: true
02:03:55.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.229446 13279 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [term 1 FOLLOWER]: Refusing update from remote peer 0f217dc377e84bf7abaadf66f72b234d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:03:55.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.211601 13508 sys_catalog.cc:455] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:55.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.232722 13496 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:03:55.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.235347 13508 sys_catalog.cc:458] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [sys.catalog]: This master's current role is: FOLLOWER
02:03:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.205902 13306 sys_catalog.cc:565] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [sys.catalog]: configured and running, proceeding with master startup.
02:03:55.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.346356 13514 sys_catalog.cc:455] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0f217dc377e84bf7abaadf66f72b234d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:55.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.347291 13514 sys_catalog.cc:458] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [sys.catalog]: This master's current role is: FOLLOWER
02:03:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.368636 13508 sys_catalog.cc:455] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f217dc377e84bf7abaadf66f72b234d. Latest consensus state: current_term: 1 leader_uuid: "0f217dc377e84bf7abaadf66f72b234d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.371608 13508 sys_catalog.cc:458] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3 [sys.catalog]: This master's current role is: FOLLOWER
02:03:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.411005 13525 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:03:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.772742 13496 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0f217dc377e84bf7abaadf66f72b234d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29d08e92eda49708a109dddae4c40b3" member_type: VOTER last_known_addr { host: "127.12.234.126" port: 34017 } } peers { permanent_uuid: "dc469968c1bd4b9bb216a6805aafd0fd" member_type: VOTER last_known_addr { host: "127.12.234.125" port: 39317 } } peers { permanent_uuid: "0f217dc377e84bf7abaadf66f72b234d" member_type: VOTER last_known_addr { host: "127.12.234.124" port: 38173 } } }
02:03:55.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.774005 13496 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f217dc377e84bf7abaadf66f72b234d [sys.catalog]: This master's current role is: LEADER
02:03:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.930578 13497 catalog_manager.cc:1294] Loaded cluster ID: 9670e36df2ca475eb34d037eef54deee
02:03:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.931432 13497 catalog_manager.cc:1587] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: loading cluster ID for follower catalog manager: success
02:03:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.940137 13497 catalog_manager.cc:1609] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: acquiring CA information for follower catalog manager: success
02:03:55.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:55.947543 13497 catalog_manager.cc:1637] T 00000000000000000000000000000000 P dc469968c1bd4b9bb216a6805aafd0fd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:03:55.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:55.960659 13509 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:55.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:55.961359 13509 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:55.962054 13509 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:56.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:56.026149 13509 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:56.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:56.027536 13509 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.65
02:03:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:56.097767 13509 tablet_server_runner.cc:78] Tablet server non-default flags:
02:03:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:0
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:03:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:03:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:03:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:56.099617 13509 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:56.102258 13509 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:56.126039 13534 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:56.437448 13525 catalog_manager.cc:1294] Loaded cluster ID: 9670e36df2ca475eb34d037eef54deee
02:03:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:56.438261 13525 catalog_manager.cc:1587] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3: loading cluster ID for follower catalog manager: success
02:03:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:56.503732 13525 catalog_manager.cc:1609] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3: acquiring CA information for follower catalog manager: success
02:03:56.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:56.555073 13525 catalog_manager.cc:1637] T 00000000000000000000000000000000 P d29d08e92eda49708a109dddae4c40b3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:03:59.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:57.525604 13533 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 13509
02:03:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.138717 13533 kernel_stack_watchdog.cc:198] Thread 13509 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 391ms:
02:03:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:03:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:03:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:03:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:03:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.139338 13509 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.008s	user 0.000s	sys 0.001s
02:03:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.139729 13509 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.008s	user 0.000s	sys 0.002s
02:03:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:56.128934 13535 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.141989 13537 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.147244 13536 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3011 milliseconds
02:03:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.147337 13509 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.149142 13509 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.151813 13509 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.153471 13509 hybrid_clock.cc:648] HybridClock initialized: now 1779069839153361 us; error 113 us; skew 500 ppm
02:03:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.154642 13509 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.163877 13509 webserver.cc:533] Webserver started at http://127.12.234.65:43241/ using document root <none> and password file <none>
02:03:59.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.165256 13509 fs_manager.cc:362] Metadata directory not provided
02:03:59.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.165570 13509 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.166172 13509 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.172989 13509 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/instance:
02:03:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22459bf8d8d44412a52e4c1805d40542"
02:03:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:59 on dist-test-slave-9xtj"
02:03:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.174716 13509 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal/instance:
02:03:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22459bf8d8d44412a52e4c1805d40542"
02:03:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:59 on dist-test-slave-9xtj"
02:03:59.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.185672 13509 fs_manager.cc:696] Time spent creating directory manager: real 0.010s	user 0.008s	sys 0.004s
02:03:59.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.194375 13547 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:03:59.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.195847 13509 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.003s
02:03:59.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.196270 13509 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:03:59.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22459bf8d8d44412a52e4c1805d40542"
02:03:59.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:59 on dist-test-slave-9xtj"
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.196621 13509 fs_report.cc:389] FS layout report
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/data
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:59.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.271468 13509 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:03:59.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.273350 13509 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.273845 13509 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:59.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.277868 13509 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:03:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.283941 13509 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:03:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.284231 13509 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
02:03:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.284472 13509 ts_tablet_manager.cc:616] Registered 0 tablets
02:03:59.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.284606 13509 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
02:03:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.510946 13509 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.65:44793
02:03:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.511155 13659 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.65:44793 every 8 connection(s)
02:03:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.513857 13509 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:03:59.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.522960 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 13509
02:03:59.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.523643 13225 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal/instance
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.561546 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:0
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:03:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.577392 13661 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:03:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.577939 13661 heartbeater.cc:461] Registering TS with master...
02:03:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.579980 13661 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:03:59.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.581974 13662 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:03:59.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.582381 13662 heartbeater.cc:461] Registering TS with master...
02:03:59.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.583833 13662 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:03:59.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.587591 13346 ts_manager.cc:194] Registered new tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:03:59.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.587585 13271 ts_manager.cc:194] Registered new tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:03:59.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.594897 13660 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:03:59.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.595399 13660 heartbeater.cc:461] Registering TS with master...
02:03:59.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.596571 13660 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:03:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.600224 13426 ts_manager.cc:194] Registered new tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:03:59.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:03:59.603116 13426 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.65:60821
02:03:59.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.937019 13670 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:59.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.937645 13670 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.938275 13670 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:59.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.980436 13670 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:03:59.981578 13670 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.66
02:04:00.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:00.025568 13670 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:0
02:04:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:00.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:04:00.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:00.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:00.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:00.027407 13670 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:00.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:00.030035 13670 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:00.047086 13676 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:00.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:00.608376 13660 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:00.050971 13677 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:01.499881 13679 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:01.503605 13678 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1447 milliseconds
02:04:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.503692 13670 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.505089 13670 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.507674 13670 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.509603 13670 hybrid_clock.cc:648] HybridClock initialized: now 1779069841509489 us; error 136 us; skew 500 ppm
02:04:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.510641 13670 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.518553 13670 webserver.cc:533] Webserver started at http://127.12.234.66:37913/ using document root <none> and password file <none>
02:04:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.519510 13670 fs_manager.cc:362] Metadata directory not provided
02:04:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.519708 13670 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.520141 13670 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.524935 13670 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/instance:
02:04:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:01 on dist-test-slave-9xtj"
02:04:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.526100 13670 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal/instance:
02:04:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:01 on dist-test-slave-9xtj"
02:04:01.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.533805 13670 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
02:04:01.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.540524 13687 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.541977 13670 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
02:04:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.542302 13670 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:01 on dist-test-slave-9xtj"
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.542629 13670 fs_report.cc:389] FS layout report
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/data
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:01.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.598045 13670 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:01.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.600324 13670 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:01.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.600777 13670 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.604769 13670 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.609753 13670 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.609988 13670 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
02:04:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.610303 13670 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.610467 13670 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
02:04:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.786804 13670 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.66:41047
02:04:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.786931 13799 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.66:41047 every 8 connection(s)
02:04:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.789315 13670 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.795261 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 13670
02:04:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.795771 13225 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal/instance
02:04:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.818233 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:0
02:04:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.840895 13800 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:01.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.841477 13800 heartbeater.cc:461] Registering TS with master...
02:04:01.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.842933 13800 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.845378 13802 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.845647 13801 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.845799 13802 heartbeater.cc:461] Registering TS with master...
02:04:01.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.846114 13801 heartbeater.cc:461] Registering TS with master...
02:04:01.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.846784 13802 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.848986 13426 ts_manager.cc:194] Registered new tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.848948 13801 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:01.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.851667 13426 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.66:56347
02:04:01.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.852669 13346 ts_manager.cc:194] Registered new tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:01.854749 13271 ts_manager.cc:194] Registered new tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.175591 13810 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.176224 13810 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:02.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.176821 13810 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.211588 13810 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.212646 13810 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.67
02:04:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:02.251314 13810 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:0
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:02.253196 13810 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:02.254760 13810 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:02.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.268625 13816 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:02.856745 13800 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:03.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.270501 13817 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:03.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:02.274513 13819 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:03.728130 13818 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1453 milliseconds
02:04:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.728283 13810 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.729616 13810 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:03.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.731861 13810 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.733296 13810 hybrid_clock.cc:648] HybridClock initialized: now 1779069843733259 us; error 85 us; skew 500 ppm
02:04:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.734170 13810 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.741928 13810 webserver.cc:533] Webserver started at http://127.12.234.67:38593/ using document root <none> and password file <none>
02:04:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.742862 13810 fs_manager.cc:362] Metadata directory not provided
02:04:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.743077 13810 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.743486 13810 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.748566 13810 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/instance:
02:04:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:03 on dist-test-slave-9xtj"
02:04:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.749835 13810 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal/instance:
02:04:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:03 on dist-test-slave-9xtj"
02:04:03.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.758862 13810 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.005s	sys 0.004s
02:04:03.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.765157 13827 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.766561 13810 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.004s
02:04:03.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.766989 13810 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:03.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:03.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:03 on dist-test-slave-9xtj"
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.767455 13810 fs_report.cc:389] FS layout report
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/data
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.831212 13810 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:03.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.833108 13810 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.833694 13810 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:03.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.837070 13810 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.842020 13810 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.842316 13810 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
02:04:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.842622 13810 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:03.842844 13810 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
02:04:04.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.023198 13810 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.67:37361
02:04:04.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.023535 13939 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.67:37361 every 8 connection(s)
02:04:04.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.027104 13810 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.037477 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 13810
02:04:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.038017 13225 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal/instance
02:04:04.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.065914 13941 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:04.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.066524 13941 heartbeater.cc:461] Registering TS with master...
02:04:04.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.067901 13941 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:04.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.070437 13940 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:04.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.070801 13940 heartbeater.cc:461] Registering TS with master...
02:04:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.071786 13940 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:04.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.072615 13346 ts_manager.cc:194] Registered new tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.074397 13426 ts_manager.cc:194] Registered new tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.075840 13942 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.076224 13942 heartbeater.cc:461] Registering TS with master...
02:04:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.076617 13426 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.67:55303
02:04:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.077159 13942 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.079886 13271 ts_manager.cc:194] Registered new tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:04.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:04.088583 13225 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:04:04.102 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
02:04:04.103 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

02:04:04.113 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.12.234.126"
      port: 34017
    }
    bound_http_address {
      host: "127.12.234.126"
      port: 42591
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.12.234.125"
      port: 39317
    }
    bound_http_address {
      host: "127.12.234.125"
      port: 44407
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.12.234.124"
      port: 38173
    }
    bound_http_address {
      host: "127.12.234.124"
      port: 46353
    }
  }
}

02:04:04.117 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

02:04:04.120 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.12.234.65"
      port: 44793
    }
    bound_http_address {
      host: "127.12.234.65"
      port: 43241
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.12.234.66"
      port: 41047
    }
    bound_http_address {
      host: "127.12.234.66"
      port: 37913
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.12.234.67"
      port: 37361
    }
    bound_http_address {
      host: "127.12.234.67"
      port: 38593
    }
  }
}

02:04:04.120 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:04:04.299 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
02:04:04.375 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
02:04:04.585 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.234.126:34017(127.12.234.126:34017)] connecting to peer
02:04:04.655 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.234.125:39317(127.12.234.125:39317)] connecting to peer
02:04:04.657 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.234.124:38173(127.12.234.124:38173)] connecting to peer
02:04:04.680 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.234.126:34017(127.12.234.126:34017)] Successfully connected to peer
02:04:04.683 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.234.125:39317(127.12.234.125:39317)] Successfully connected to peer
02:04:04.687 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.12.234.124:38173(127.12.234.124:38173)] Successfully connected to peer
02:04:04.796 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.234.125
02:04:04.797 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.234.126
02:04:04.798 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.12.234.124
02:04:05.044 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc59a1b17, L:/127.0.0.1:38226 - R:/127.12.234.125:39317] using SASL/PLAIN
02:04:05.044 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xb2f9e37f, L:/127.0.0.1:55924 - R:/127.12.234.124:38173] using SASL/PLAIN
02:04:05.045 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3a35ad05, L:/127.0.0.1:49086 - R:/127.12.234.126:34017] using SASL/PLAIN
02:04:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.081668 13940 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:05.090 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:04:05.091 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=550), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.234.126:34017, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.234.125:39317, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.234.124:38173, [517ms] Sub RPC ConnectToMaster: received response from server master-127.12.234.126:34017: OK, [517ms] Sub RPC ConnectToMaster: received response from server master-127.12.234.125:39317: OK, [518ms] Sub RPC ConnectToMaster: received response from server master-127.12.234.124:38173: OK, deferred=null) after lookup
02:04:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.144205 13426 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55924:
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523"
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:04:05.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:04:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:04:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:04:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
02:04:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
02:04:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
02:04:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
02:04:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
02:04:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:04:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:05.149032 13426 catalog_manager.cc:7058] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:04:05.229 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table a8349a72522d4e3bb3a4a2056acde290
02:04:05.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.266850 13875 tablet_service.cc:1511] Processing CreateTablet for tablet 1aab78234d3348cabb4f11d3aee773bc (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523 [id=a8349a72522d4e3bb3a4a2056acde290]), partition=RANGE (key) PARTITION UNBOUNDED
02:04:05.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.269841 13875 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1aab78234d3348cabb4f11d3aee773bc. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:05.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.275696 13595 tablet_service.cc:1511] Processing CreateTablet for tablet 1aab78234d3348cabb4f11d3aee773bc (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523 [id=a8349a72522d4e3bb3a4a2056acde290]), partition=RANGE (key) PARTITION UNBOUNDED
02:04:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.280122 13595 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1aab78234d3348cabb4f11d3aee773bc. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:05.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.284884 13735 tablet_service.cc:1511] Processing CreateTablet for tablet 1aab78234d3348cabb4f11d3aee773bc (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523 [id=a8349a72522d4e3bb3a4a2056acde290]), partition=RANGE (key) PARTITION UNBOUNDED
02:04:05.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.287258 13735 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1aab78234d3348cabb4f11d3aee773bc. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.303094 13975 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap starting.
02:04:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.311487 13975 tablet_bootstrap.cc:654] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Neither blocks nor log segments found. Creating new log.
02:04:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.313679 13975 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Log is configured to *not* fsync() on all Append() calls
02:04:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.320333 13975 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: No bootstrap required, opened a new log
02:04:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.321226 13975 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent bootstrapping tablet: real 0.020s	user 0.012s	sys 0.004s
02:04:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.324622 13977 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap starting.
02:04:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.328625 13978 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap starting.
02:04:05.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.338027 13977 tablet_bootstrap.cc:654] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Neither blocks nor log segments found. Creating new log.
02:04:05.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.338025 13978 tablet_bootstrap.cc:654] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Neither blocks nor log segments found. Creating new log.
02:04:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.341022 13978 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Log is configured to *not* fsync() on all Append() calls
02:04:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.341392 13977 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Log is configured to *not* fsync() on all Append() calls
02:04:05.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.354231 13978 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: No bootstrap required, opened a new log
02:04:05.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.354875 13978 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent bootstrapping tablet: real 0.027s	user 0.018s	sys 0.004s
02:04:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.360980 13975 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.362360 13975 raft_consensus.cc:385] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.362912 13975 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Initialized, Role: FOLLOWER
02:04:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.364270 13975 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.367646 13977 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: No bootstrap required, opened a new log
02:04:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.368822 13977 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent bootstrapping tablet: real 0.045s	user 0.017s	sys 0.005s
02:04:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.379000 13975 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent starting tablet: real 0.057s	user 0.038s	sys 0.007s
02:04:05.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.403939 13977 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.405385 13977 raft_consensus.cc:385] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:05.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.405927 13977 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Initialized, Role: FOLLOWER
02:04:05.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.406895 13978 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.407831 13977 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.408823 13978 raft_consensus.cc:385] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.409154 13978 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22459bf8d8d44412a52e4c1805d40542, State: Initialized, Role: FOLLOWER
02:04:05.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.411108 13978 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.418329 13977 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent starting tablet: real 0.049s	user 0.030s	sys 0.013s
02:04:05.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.429988 13978 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent starting tablet: real 0.075s	user 0.044s	sys 0.007s
02:04:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:05.548056 13663 tablet.cc:2404] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:05.549257 13947 tablet.cc:2404] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.552865 13982 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:05.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.553579 13982 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.556273 13982 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:05.571949 13803 tablet.cc:2404] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:05.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.572994 13895 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256" is_pre_election: true
02:04:05.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.573172 13615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542" is_pre_election: true
02:04:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.574433 13895 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 0.
02:04:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.574550 13615 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 0.
02:04:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.576035 13689 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 
02:04:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.576906 13982 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:04:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.577281 13982 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:04:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.577579 13982 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 0 FOLLOWER]: Advancing to term 1
02:04:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.583217 13982 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.585647 13982 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 1 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.586697 13895 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.586795 13615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.587299 13615 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 0 FOLLOWER]: Advancing to term 1
02:04:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.587291 13895 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 0 FOLLOWER]: Advancing to term 1
02:04:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.595075 13615 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 1.
02:04:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.595081 13895 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 1.
02:04:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.596668 13691 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256; no voters: 
02:04:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.597522 13982 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 1 FOLLOWER]: Leader election won for term 1
02:04:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.599393 13982 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 1 LEADER]: Becoming Leader. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Running, Role: LEADER
02:04:05.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.600375 13982 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:05.613404 13426 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66). New cstate: current_term: 1 leader_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: HEALTHY } } }
02:04:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:06.073246 13982 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:04:06.075 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table a8349a72522d4e3bb3a4a2056acde290
02:04:06.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:06.075031 13982 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:04:06.103 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:06.118 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299986, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:06.121 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [12ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=null) after lookup
02:04:06.146 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:06.160 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] Successfully connected to peer
02:04:06.241 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x2c44ed1c, L:/127.0.0.1:59068 - R:/127.12.234.66:41047] using TOKEN/null
02:04:06.465 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:06.469 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.66:41047
02:04:06.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

02:04:06.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:06.471982 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 13670
02:04:06.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:06.501 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] handling channelInactive
02:04:06.502 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state READY due to: connection closed
02:04:06.503 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: connection closed
02:04:06.528 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:06.529 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:06.529 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test)) after lookup
02:04:06.530 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:06.533 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.533 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.534 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.548 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:06.549 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:06.549 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=50), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [35ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [35ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [49ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328))) after lookup
02:04:06.550 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:06.555 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.555 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.556 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.608 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:06.608 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:06.609 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=109), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [35ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [35ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [49ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [51ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [57ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [57ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797))) after lookup
02:04:06.610 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:06.613 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.614 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.614 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.667 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:06.667 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:06.668 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=168), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [35ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [35ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [49ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [51ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [57ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [57ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [111ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [115ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [115ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [167ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348))) after lookup
02:04:06.669 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:06.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.671 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.672 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.967 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:06.968 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:06.968 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=468), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [35ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [35ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [49ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [51ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [57ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [57ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [111ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [115ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [115ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [167ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [169ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [172ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [173ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [463ms] refreshing cache from master, [463ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [468ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605))) after lookup
02:04:06.970 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:06.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:06.972 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:07.788 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:07.788 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:07.789 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1289), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [35ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [35ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [49ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [51ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [57ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [57ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [111ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [115ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [115ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [167ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [169ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [172ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [173ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [463ms] refreshing cache from master, [463ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [468ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [470ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [473ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [473ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [1283ms] refreshing cache from master, [1283ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1288ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605) -> (continuation of Deferred@1661984934 after retry RPC@764936462), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605) -> (continuation of Deferred@1661984934 after retry RPC@764936462))) after lookup
02:04:07.791 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:07.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:07.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:07.792 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.845288 14011 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.845837 14011 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.848783 14012 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.849756 14012 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.848869 14011 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:07.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:07.854532 13549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:07.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.855012 14012 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:07.867208 13829 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:07.871026 13549 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:07.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:07.876233 13829 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.876359 13895 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256" is_pre_election: true
02:04:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.876772 13615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542" is_pre_election: true
02:04:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.876947 13895 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22459bf8d8d44412a52e4c1805d40542 in term 1.
02:04:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.877437 13615 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 1.
02:04:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.878410 13551 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:07.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.879361 14011 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:04:07.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.879750 14011 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:07.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.880059 14011 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 1 FOLLOWER]: Advancing to term 2
02:04:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.880266 13829 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.881712 14012 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.882117 14012 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.882511 14012 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 1 FOLLOWER]: Advancing to term 2
02:04:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.888478 14011 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.890491 14011 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 2 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.890869 14012 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.891279 13895 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.892715 13895 raft_consensus.cc:2393] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 22459bf8d8d44412a52e4c1805d40542 in current term 2: Already voted for candidate 7f3c1e9950ff41b09847d354de219256 in this term.
02:04:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.895165 14012 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 2 election: Requested vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:07.895620 13549 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.896041 13549 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256
02:04:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.896884 14011 raft_consensus.cc:2749] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
02:04:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.896761 13615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.897970 13615 raft_consensus.cc:2393] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 7f3c1e9950ff41b09847d354de219256 in current term 2: Already voted for candidate 22459bf8d8d44412a52e4c1805d40542 in this term.
02:04:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:07.901799 13829 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.902150 13829 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f3c1e9950ff41b09847d354de219256; no voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:07.902822 14012 raft_consensus.cc:2749] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
02:04:08.787 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:08.787 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:08.789 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2288), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [35ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [35ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [49ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [51ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [57ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [57ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [111ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [115ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [115ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [167ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [169ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [172ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [173ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [463ms] refreshing cache from master, [463ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [468ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [470ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [473ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [473ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [1283ms] refreshing cache from master, [1283ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1288ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [1291ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [1293ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [1293ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2283ms] refreshing cache from master, [2283ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [2288ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605) -> (continuation of Deferred@1661984934 after retry RPC@764936462) -> (continuation of Deferred@1717499441 after retry RPC@186846085), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605) -> (continuation of Deferred@1661984934 after retry RPC@764936462) -> (continuation of Deferred@1717499441 after retry RPC@186846085))) after lookup
02:04:08.790 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:08.791 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:08.792 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:08.792 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.755622 14024 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.756045 14024 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:09.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.757611 14024 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:09.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.758725 13615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542" is_pre_election: true
02:04:09.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.759244 13615 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 2.
02:04:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.760896 13829 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 
02:04:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.761595 14024 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Leader pre-election won for term 3
02:04:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.761883 14024 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:04:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.762238 14024 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 2 FOLLOWER]: Advancing to term 3
02:04:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:09.762648 13829 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:09.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.766637 14024 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.768491 14024 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 3 election: Requested vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.769398 13615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.769975 13615 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 2 FOLLOWER]: Advancing to term 3
02:04:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:09.772646 13829 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:09.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.775074 13615 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 3.
02:04:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.776089 13829 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:09.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.776856 14024 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 3 FOLLOWER]: Leader election won for term 3
02:04:09.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.777865 14024 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 3 LEADER]: Becoming Leader. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Running, Role: LEADER
02:04:09.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.779211 14024 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:09.788240 13425 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 reported cstate change: term changed from 1 to 3, leader changed from 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66) to 7f3c1e9950ff41b09847d354de219256 (127.12.234.67). New cstate: current_term: 3 leader_uuid: "7f3c1e9950ff41b09847d354de219256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: UNKNOWN } } }
02:04:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:10.196753 13615 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 3 FOLLOWER]: Refusing update from remote peer 7f3c1e9950ff41b09847d354de219256: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
02:04:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:10.198220 14024 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
02:04:10.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:10.218609 13829 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 -> Peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Couldn't send request to peer 473a4c326e6c4936ad3fcff3ade5d4d9. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:12.287 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:12.288 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:12.288 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=16, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=5789), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [3ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [23ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [29ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [35ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [35ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [43ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [49ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [51ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [57ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [57ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [103ms] refreshing cache from master, [103ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [108ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [111ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [115ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [115ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [167ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [169ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [172ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [173ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [463ms] refreshing cache from master, [463ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [468ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [470ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [473ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [473ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [1283ms] refreshing cache from master, [1283ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1288ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [1291ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [1293ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [1293ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2283ms] refreshing cache from master, [2283ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [2288ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [2290ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2293ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2293ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [5783ms] refreshing cache from master, [5784ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [5788ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1324443568(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1cb62ca2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605) -> (continuation of Deferred@1661984934 after retry RPC@764936462) -> (continuation of Deferred@1717499441 after retry RPC@186846085) -> (continuation of Deferred@1831999653 after retry RPC@1278412037), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12dd07a2 -> wakeup thread Time-limited test -> (continuation of Deferred@499446075 after retry RPC@1553745328) -> (continuation of Deferred@41329988 after retry RPC@858718797) -> (continuation of Deferred@1323444628 after retry RPC@15812348) -> (continuation of Deferred@1018313603 after retry RPC@406506605) -> (continuation of Deferred@1661984934 after retry RPC@764936462) -> (continuation of Deferred@1717499441 after retry RPC@186846085) -> (continuation of Deferred@1831999653 after retry RPC@1278412037))) after lookup
02:04:12.290 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:12.291 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] Successfully connected to peer
02:04:12.325 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xae016594, L:/127.0.0.1:38860 - R:/127.12.234.67:37361] using TOKEN/null
02:04:12.392 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.66:41047
02:04:12.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:12.396136 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:41047
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.583197 13829 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 -> Peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Couldn't send request to peer 473a4c326e6c4936ad3fcff3ade5d4d9. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.751994 14061 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.752655 14061 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:12.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.753312 14061 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.792750 14061 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:12.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.793810 14061 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.66
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:12.835727 14061 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:41047
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:12.837839 14061 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:12.839872 14061 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:12.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.855289 14068 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:13.144301 13829 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111) [suppressed 9 similar messages]
02:04:14.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:14.261632 14067 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 14061
02:04:14.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:14.455411 14061 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.600s	user 0.518s	sys 1.020s
02:04:14.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:14.455830 14061 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.601s	user 0.518s	sys 1.020s
02:04:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:12.855520 14069 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:14.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:14.458375 14071 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:14.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:14.463474 14070 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1602 milliseconds
02:04:14.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.463590 14061 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:14.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.465015 14061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:14.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.467489 14061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:14.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.468968 14061 hybrid_clock.cc:648] HybridClock initialized: now 1779069854468920 us; error 39 us; skew 500 ppm
02:04:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.470018 14061 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.477005 14061 webserver.cc:533] Webserver started at http://127.12.234.66:37913/ using document root <none> and password file <none>
02:04:14.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.478000 14061 fs_manager.cc:362] Metadata directory not provided
02:04:14.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.478235 14061 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:14.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.486927 14061 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.002s
02:04:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.492265 14082 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:14.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.494045 14061 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.001s
02:04:14.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.494402 14061 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:14.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:14.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:01 on dist-test-slave-9xtj"
02:04:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.496564 14061 fs_report.cc:389] FS layout report
02:04:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/data
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.589949 14061 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:14.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.591475 14061 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.591904 14061 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:14.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.594755 14061 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.601320 14089 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.610688 14061 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.610955 14061 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
02:04:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.611212 14061 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.616264 14061 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.616533 14061 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.006s	sys 0.000s
02:04:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.617097 14089 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap starting.
02:04:14.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.688285 14089 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Log is configured to *not* fsync() on all Append() calls
02:04:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.731140 14089 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:14.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.732457 14089 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap complete.
02:04:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.734596 14089 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent bootstrapping tablet: real 0.118s	user 0.078s	sys 0.034s
02:04:14.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.772275 14089 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.776604 14089 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Initialized, Role: FOLLOWER
02:04:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.777973 14089 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.784186 14089 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent starting tablet: real 0.049s	user 0.037s	sys 0.012s
02:04:14.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.803416 14061 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.66:41047
02:04:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.803766 14199 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.66:41047 every 8 connection(s)
02:04:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.806274 14061 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.809659 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 14061
02:04:14.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:14.856 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:14.858 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.67:37361
02:04:14.859 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:14.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.860140 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 13810
02:04:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.866122 14200 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.866632 14200 heartbeater.cc:461] Registering TS with master...
02:04:14.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.867776 14200 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:14.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.869694 14201 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:14.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.870882 14201 heartbeater.cc:461] Registering TS with master...
02:04:14.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.872030 14201 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:14.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.874820 13425 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.876596 13344 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.878528 14202 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:14.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.878898 14202 heartbeater.cc:461] Registering TS with master...
02:04:14.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.878830 13425 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.66:34521
02:04:14.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.879891 14202 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:14.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:14.884332 13271 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:14.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:14.897 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] handling channelInactive
02:04:14.897 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state READY due to: connection closed
02:04:14.903 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:14.905 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.905 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.905 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.927 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:14.927 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:14.927 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=3, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test)) after lookup
02:04:14.928 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:14.929 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.930 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.930 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.947 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:14.948 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:14.948 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=5, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=51), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [50ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229))) after lookup
02:04:14.950 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:14.951 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.951 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.951 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.968 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:14.968 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:14.969 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=7, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [50ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [52ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [54ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [54ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029))) after lookup
02:04:14.969 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:14.971 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.971 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:14.972 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.227 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:15.228 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:15.228 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=9, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=331), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [50ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [52ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [54ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [54ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [74ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [75ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [325ms] refreshing cache from master, [325ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [330ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916))) after lookup
02:04:15.230 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:15.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.231 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.232 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.708 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:15.709 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:15.709 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=11, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=812), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [50ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [52ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [54ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [54ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [74ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [75ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [325ms] refreshing cache from master, [325ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [330ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [332ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [334ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [335ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [805ms] refreshing cache from master, [805ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [811ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899))) after lookup
02:04:15.710 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:15.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.713 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:15.883443 14200 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:15.888 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:15.888 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:15.889 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=13, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=991), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [50ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [52ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [54ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [54ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [74ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [75ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [325ms] refreshing cache from master, [325ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [330ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [332ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [334ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [335ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [805ms] refreshing cache from master, [805ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [811ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [813ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [816ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [816ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [985ms] refreshing cache from master, [985ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [990ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899) -> (continuation of Deferred@1534539277 after retry RPC@606861225), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899) -> (continuation of Deferred@1534539277 after retry RPC@606861225))) after lookup
02:04:15.891 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:15.892 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.892 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:15.893 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:16.227 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:16.227 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:16.228 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=15, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1330), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [50ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [52ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [54ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [54ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [74ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [75ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [325ms] refreshing cache from master, [325ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [330ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [332ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [334ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [335ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [805ms] refreshing cache from master, [805ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [811ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [813ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [816ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [816ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [985ms] refreshing cache from master, [985ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [990ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [993ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [996ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [996ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [1325ms] refreshing cache from master, [1325ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1330ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899) -> (continuation of Deferred@1534539277 after retry RPC@606861225) -> (continuation of Deferred@2059518036 after retry RPC@770252895), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899) -> (continuation of Deferred@1534539277 after retry RPC@606861225) -> (continuation of Deferred@2059518036 after retry RPC@770252895))) after lookup
02:04:16.229 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:16.230 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:16.230 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:16.230 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:16.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.236938 14212 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.237650 14212 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:16.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.239688 14212 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:16.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:16.242918 13551 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:04:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:16.248358 13551 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.254746 14152 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" is_pre_election: true
02:04:16.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.255708 14152 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22459bf8d8d44412a52e4c1805d40542 in term 1.
02:04:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.257273 13549 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.257961 14212 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 3 FOLLOWER]: Leader pre-election won for term 4
02:04:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.258342 14212 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:16.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.258886 14212 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 3 FOLLOWER]: Advancing to term 4
02:04:16.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.264573 14212 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.266409 14212 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 4 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.267494 14152 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 4 candidate_status { last_received { term: 3 index: 8 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:16.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.267937 14152 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 1 FOLLOWER]: Advancing to term 4
02:04:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:16.270823 13551 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.276116 14152 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22459bf8d8d44412a52e4c1805d40542 in term 4.
02:04:16.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.278764 13549 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:16.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.279564 14212 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 4 FOLLOWER]: Leader election won for term 4
02:04:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.280874 14212 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 4 LEADER]: Becoming Leader. State: Replica: 22459bf8d8d44412a52e4c1805d40542, State: Running, Role: LEADER
02:04:16.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.282980 14212 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:16.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.293274 13426 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 reported cstate change: term changed from 3 to 4, leader changed from 7f3c1e9950ff41b09847d354de219256 (127.12.234.67) to 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65). New cstate: current_term: 4 leader_uuid: "22459bf8d8d44412a52e4c1805d40542" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: UNKNOWN } } }
02:04:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:16.666771 13551 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:16.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.742365 14152 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 4 FOLLOWER]: Refusing update from remote peer 22459bf8d8d44412a52e4c1805d40542: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 4 index: 9. (index mismatch)
02:04:16.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:16.744223 14212 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [LEADER]: Connected to new peer: Peer: permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
02:04:18.068 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:18.068 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:18.069 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=17, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=3171), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [0ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [8ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [8ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [45ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [50ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [52ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [54ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [54ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [65ms] refreshing cache from master, [65ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [74ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [75ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [325ms] refreshing cache from master, [325ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [330ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [332ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [334ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [335ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [805ms] refreshing cache from master, [805ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [811ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [813ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [816ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [816ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [985ms] refreshing cache from master, [985ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [990ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [993ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [996ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [996ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [1325ms] refreshing cache from master, [1325ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1330ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [1331ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [1333ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [1334ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3165ms] refreshing cache from master, [3165ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [3170ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1090378551(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@1432f500 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899) -> (continuation of Deferred@1534539277 after retry RPC@606861225) -> (continuation of Deferred@2059518036 after retry RPC@770252895) -> (continuation of Deferred@1268040070 after retry RPC@798747507), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@680dadbd -> wakeup thread Time-limited test -> (continuation of Deferred@1085557051 after retry RPC@1162966229) -> (continuation of Deferred@1542390305 after retry RPC@366355029) -> (continuation of Deferred@1966728023 after retry RPC@683016916) -> (continuation of Deferred@882009559 after retry RPC@740974899) -> (continuation of Deferred@1534539277 after retry RPC@606861225) -> (continuation of Deferred@2059518036 after retry RPC@770252895) -> (continuation of Deferred@1268040070 after retry RPC@798747507))) after lookup
02:04:18.070 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:18.071 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] Successfully connected to peer
02:04:18.122 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0xe42b385c, L:/127.0.0.1:57454 - R:/127.12.234.65:44793] using TOKEN/null
02:04:18.184 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.67:37361
02:04:18.186 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:18.187848 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:18.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.544840 14250 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:18.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.545344 14250 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:18.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.545794 14250 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:18.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.585278 14250 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.586434 14250 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.67
02:04:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:18.627177 14250 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:18.628748 14250 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:18.630460 14250 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:18.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.645984 14257 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:18.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.951999 13551 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:19.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.648481 14258 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:19.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:19.913340 14259 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:04:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.914002 14250 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:18.651664 14260 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:19.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.919970 14250 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:19.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.922991 14250 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:19.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.924448 14250 hybrid_clock.cc:648] HybridClock initialized: now 1779069859924399 us; error 48 us; skew 500 ppm
02:04:19.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.925459 14250 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:19.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.931943 14250 webserver.cc:533] Webserver started at http://127.12.234.67:38593/ using document root <none> and password file <none>
02:04:19.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.933154 14250 fs_manager.cc:362] Metadata directory not provided
02:04:19.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.933444 14250 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:19.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.942883 14250 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.001s	sys 0.004s
02:04:19.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.948094 14267 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.949587 14250 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
02:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.949973 14250 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:03 on dist-test-slave-9xtj"
02:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:19.952092 14250 fs_report.cc:389] FS layout report
02:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/data
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:20.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.016762 14250 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:20.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.018313 14250 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:20.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.018764 14250 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:20.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.021543 14250 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.027894 14274 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.037533 14250 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:20.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.037781 14250 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
02:04:20.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.038017 14250 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.043511 14250 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.043800 14250 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.003s	sys 0.000s
02:04:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.044248 14274 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap starting.
02:04:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.135900 14274 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Log is configured to *not* fsync() on all Append() calls
02:04:20.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.180444 14274 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:20.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.181485 14274 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap complete.
02:04:20.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.183133 14274 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent bootstrapping tablet: real 0.139s	user 0.087s	sys 0.036s
02:04:20.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.203647 14274 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:20.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.206640 14274 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Initialized, Role: FOLLOWER
02:04:20.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.208551 14274 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 3.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:20.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.213778 14274 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent starting tablet: real 0.030s	user 0.024s	sys 0.008s
02:04:20.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.241716 14250 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.67:37361
02:04:20.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.242049 14382 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.67:37361 every 8 connection(s)
02:04:20.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.244920 14250 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:20.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.248210 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 14250
02:04:20.249 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:20.296 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.291350 14384 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.291817 14384 heartbeater.cc:461] Registering TS with master...
02:04:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.293252 14384 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.297240 14337 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 3 FOLLOWER]: Advancing to term 4
02:04:20.298 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.65:44793
02:04:20.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

02:04:20.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.302045 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 13509
02:04:20.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.305248 14385 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.306084 14385 heartbeater.cc:461] Registering TS with master...
02:04:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.307188 14385 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.307971 13426 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.310987 13426 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.67:45539
02:04:20.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.315207 14384 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.324862 14388 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.325316 14388 heartbeater.cc:461] Registering TS with master...
02:04:20.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.326052 14388 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:20.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:20.346735 14269 connection.cc:465] server connection from 127.12.234.65:53589 torn down before Call kudu.consensus.ConsensusService.UpdateConsensus from 127.12.234.65:53589 (request call id 0) could send its response
02:04:20.348 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] handling channelInactive
02:04:20.349 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state READY due to: connection closed
02:04:20.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:20.351 [DEBUG - Time-limited test] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:20.353 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.354910 13344 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:20.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:20.355859 13271 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:20.362 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.362 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.388 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:20.388 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:20.389 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [11ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [11ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [37ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@779210150(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e417b68 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a66f33 -> wakeup thread Time-limited test)) after lookup
02:04:20.390 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:20.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.391 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.407 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:20.407 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:20.408 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [11ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [11ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [37ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [38ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [40ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [40ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [55ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@779210150(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e417b68 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a66f33 -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891))) after lookup
02:04:20.409 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:20.412 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.412 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.412 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.447 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:20.447 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:20.447 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=97), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [11ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [11ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [37ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [38ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [40ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [40ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [55ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [58ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [61ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [61ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [95ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@779210150(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e417b68 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a66f33 -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663))) after lookup
02:04:20.448 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:20.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.450 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.486 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:20.487 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:20.487 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=136), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [11ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [11ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [37ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [38ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [40ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [40ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [55ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [58ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [61ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [61ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [95ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [97ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [99ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [99ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [135ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@779210150(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e417b68 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a66f33 -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893))) after lookup
02:04:20.488 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:20.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.490 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.527 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:20.527 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:20.530 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=179), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [11ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [11ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [37ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [38ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [40ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [40ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [55ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [58ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [61ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [61ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [95ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [97ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [99ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [99ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [135ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [137ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [139ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [139ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [175ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@779210150(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e417b68 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893) -> (continuation of Deferred@983924915 after retry RPC@1164731192), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a66f33 -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893) -> (continuation of Deferred@983924915 after retry RPC@1164731192))) after lookup
02:04:20.531 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:20.532 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.532 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.532 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.888 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:20.888 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:20.889 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=538), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [11ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [11ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [37ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [38ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [40ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [40ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [55ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [58ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [61ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [61ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [95ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [97ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [99ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [99ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [135ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [137ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [139ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [139ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [175ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [179ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [181ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [181ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [531ms] refreshing cache from master, [532ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [536ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@779210150(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e417b68 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893) -> (continuation of Deferred@983924915 after retry RPC@1164731192) -> (continuation of Deferred@639010844 after retry RPC@1585177277), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a66f33 -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893) -> (continuation of Deferred@983924915 after retry RPC@1164731192) -> (continuation of Deferred@639010844 after retry RPC@1585177277))) after lookup
02:04:20.890 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:20.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:20.892 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:21.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.674836 14400 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 22459bf8d8d44412a52e4c1805d40542)
02:04:21.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.675328 14400 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.677619 14400 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:21.681982 14084 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:21.685999 14084 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.690825 14337 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256" is_pre_election: true
02:04:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.693112 14086 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256
02:04:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.693850 14400 raft_consensus.cc:2749] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
02:04:21.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.815163 14405 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 22459bf8d8d44412a52e4c1805d40542)
02:04:21.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.815630 14405 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:21.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.817835 14405 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:21.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:21.822952 14269 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:21.827651 14269 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:21.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.832150 14152 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" is_pre_election: true
02:04:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.832846 14152 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 4.
02:04:21.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.833961 14269 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256; no voters: 22459bf8d8d44412a52e4c1805d40542
02:04:21.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.834790 14405 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 4 FOLLOWER]: Leader pre-election won for term 5
02:04:21.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.835095 14405 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 22459bf8d8d44412a52e4c1805d40542)
02:04:21.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.835513 14405 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 4 FOLLOWER]: Advancing to term 5
02:04:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.841434 14405 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.843197 14405 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 5 election: Requested vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.845045 14152 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.845546 14152 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 4 FOLLOWER]: Advancing to term 5
02:04:21.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:21.846606 14269 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.850745 14152 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 5.
02:04:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.851902 14269 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256; no voters: 22459bf8d8d44412a52e4c1805d40542
02:04:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.852895 14405 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 5 FOLLOWER]: Leader election won for term 5
02:04:21.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.854425 14405 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 5 LEADER]: Becoming Leader. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Running, Role: LEADER
02:04:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.855441 14405 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:21.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:21.865041 13426 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 reported cstate change: term changed from 4 to 5, leader changed from 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65) to 7f3c1e9950ff41b09847d354de219256 (127.12.234.67). New cstate: current_term: 5 leader_uuid: "7f3c1e9950ff41b09847d354de219256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: UNKNOWN } } }
02:04:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:22.237815 14269 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 -> Peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Couldn't send request to peer 22459bf8d8d44412a52e4c1805d40542. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:22.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:22.324453 14152 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 5 FOLLOWER]: Refusing update from remote peer 7f3c1e9950ff41b09847d354de219256: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
02:04:22.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:22.326014 14405 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Connected to new peer: Peer: permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
02:04:22.889 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:22.889 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:22.890 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=2539), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [11ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [11ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [37ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [38ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [40ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [40ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [55ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [58ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [61ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [61ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [91ms] refreshing cache from master, [91ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [95ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [97ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [99ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [99ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [131ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [135ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [137ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [139ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [139ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [171ms] refreshing cache from master, [171ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [175ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [179ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [181ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [181ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [531ms] refreshing cache from master, [532ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [536ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [538ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [541ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [541ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2532ms] refreshing cache from master, [2532ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [2538ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@779210150(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e417b68 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893) -> (continuation of Deferred@983924915 after retry RPC@1164731192) -> (continuation of Deferred@639010844 after retry RPC@1585177277) -> (continuation of Deferred@1853707302 after retry RPC@1871850846), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@72a66f33 -> wakeup thread Time-limited test -> (continuation of Deferred@1641177213 after retry RPC@550261891) -> (continuation of Deferred@1502968354 after retry RPC@486817663) -> (continuation of Deferred@1982176369 after retry RPC@1256634893) -> (continuation of Deferred@983924915 after retry RPC@1164731192) -> (continuation of Deferred@639010844 after retry RPC@1585177277) -> (continuation of Deferred@1853707302 after retry RPC@1871850846))) after lookup
02:04:22.891 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:22.892 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] Successfully connected to peer
02:04:22.913 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0xca6eb997, L:/127.0.0.1:49546 - R:/127.12.234.67:37361] using TOKEN/null
02:04:22.968 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.65:44793
02:04:22.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:22.969822 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:44793
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43241
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:23.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:23.314018 14436 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:23.314647 14436 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:23.315146 14436 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:23.359710 14436 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:23.361361 14436 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.65
02:04:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:23.404098 14436 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:44793
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43241
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:23.405926 14436 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:23.407873 14436 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:23.422536 14444 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:24.743324 14269 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 -> Peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Couldn't send request to peer 22459bf8d8d44412a52e4c1805d40542. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:24.825132 14443 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 14436
02:04:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:24.926059 14436 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.504s	user 0.565s	sys 0.910s
02:04:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:24.926499 14436 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.504s	user 0.565s	sys 0.910s
02:04:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:23.423312 14445 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:24.928907 14447 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:24.932518 14446 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1504 milliseconds
02:04:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.932727 14436 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.934059 14436 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:24.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.936846 14436 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.938475 14436 hybrid_clock.cc:648] HybridClock initialized: now 1779069864938388 us; error 126 us; skew 500 ppm
02:04:24.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.939771 14436 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.946987 14436 webserver.cc:533] Webserver started at http://127.12.234.65:43241/ using document root <none> and password file <none>
02:04:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.948490 14436 fs_manager.cc:362] Metadata directory not provided
02:04:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.948803 14436 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.957720 14436 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.008s	sys 0.000s
02:04:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.964458 14458 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:24.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.965873 14436 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.007s	sys 0.000s
02:04:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.966218 14436 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:59 on dist-test-slave-9xtj"
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:24.968372 14436 fs_report.cc:389] FS layout report
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/data
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.037374 14436 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.039176 14436 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.039906 14436 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:25.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.043057 14436 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.049906 14465 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.058740 14436 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.059052 14436 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
02:04:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.059487 14436 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.065145 14436 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:25.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.065442 14436 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.004s	sys 0.002s
02:04:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.066011 14465 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap starting.
02:04:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.147904 14465 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Log is configured to *not* fsync() on all Append() calls
02:04:25.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.227959 14465 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.229503 14465 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap complete.
02:04:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.232033 14465 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent bootstrapping tablet: real 0.167s	user 0.123s	sys 0.035s
02:04:25.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.255398 14436 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.65:44793
02:04:25.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.255625 14572 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.65:44793 every 8 connection(s)
02:04:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.259428 14436 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:25.260 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.260066 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 14436
02:04:25.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.264495 14465 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.269768 14465 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22459bf8d8d44412a52e4c1805d40542, State: Initialized, Role: FOLLOWER
02:04:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.273963 14465 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.291646 14465 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent starting tablet: real 0.059s	user 0.033s	sys 0.013s
02:04:25.298 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:25.322 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.67:37361
02:04:25.323 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:25.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.323510 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 14250
02:04:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.358613 14575 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.359133 14575 heartbeater.cc:461] Registering TS with master...
02:04:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.360589 14575 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:25.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.365700 13344 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:25.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.397176 14573 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:25.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.397626 14573 heartbeater.cc:461] Registering TS with master...
02:04:25.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.398628 14573 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.402534 13425 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.406721 13425 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.65:47359
02:04:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.409274 14583 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.409682 14583 heartbeater.cc:461] Registering TS with master...
02:04:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.410602 14583 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:25.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.413184 14573 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:25.416694 13271 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:25.427 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] handling channelInactive
02:04:25.428 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state READY due to: connection closed
02:04:25.429 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:25.430 [DEBUG - Time-limited test] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:25.431 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.432 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.432 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.447 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:25.448 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:25.449 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [17ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@935262947(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@49b8d997 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7938168b -> wakeup thread Time-limited test)) after lookup
02:04:25.450 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:25.451 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.451 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.451 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.466 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:25.467 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:25.467 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [17ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [19ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [21ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [21ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@935262947(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@49b8d997 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7938168b -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855))) after lookup
02:04:25.467 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:25.468 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.468 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.468 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.507 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:25.508 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:25.508 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [17ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [19ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [21ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [21ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [37ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [38ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [39ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [77ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@935262947(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@49b8d997 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7938168b -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638))) after lookup
02:04:25.509 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:25.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.511 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.626 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:25.627 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:25.627 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=197), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [17ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [19ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [21ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [21ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [37ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [38ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [39ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [77ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [79ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [81ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [81ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [196ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@935262947(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@49b8d997 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7938168b -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408))) after lookup
02:04:25.627 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:25.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:25.629 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:26.067 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:26.068 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:26.068 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=638), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [17ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [19ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [21ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [21ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [37ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [38ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [39ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [77ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [79ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [81ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [81ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [196ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [197ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [199ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [199ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [633ms] refreshing cache from master, [633ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [637ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@935262947(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@49b8d997 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408) -> (continuation of Deferred@840060666 after retry RPC@278077284), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7938168b -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408) -> (continuation of Deferred@840060666 after retry RPC@278077284))) after lookup
02:04:26.069 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:26.070 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:26.070 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:26.070 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:26.367 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:26.368 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:26.368 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=938), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [17ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [19ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [21ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [21ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [37ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [38ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [39ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [77ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [79ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [81ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [81ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [196ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [197ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [199ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [199ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [633ms] refreshing cache from master, [633ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [637ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [639ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [640ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [640ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [932ms] refreshing cache from master, [932ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [937ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@935262947(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@49b8d997 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408) -> (continuation of Deferred@840060666 after retry RPC@278077284) -> (continuation of Deferred@749571520 after retry RPC@1138930614), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7938168b -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408) -> (continuation of Deferred@840060666 after retry RPC@278077284) -> (continuation of Deferred@749571520 after retry RPC@1138930614))) after lookup
02:04:26.370 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:26.371 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:26.371 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:26.371 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.533092 14587 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.533794 14587 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:26.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.535795 14587 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:26.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:26.545379 14086 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.550647 14528 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542" is_pre_election: true
02:04:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.551357 14528 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 4.
02:04:26.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.552881 14084 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:26.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.553683 14587 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 5 FOLLOWER]: Leader pre-election won for term 6
02:04:26.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.553998 14587 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:26.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.554397 14587 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 5 FOLLOWER]: Advancing to term 6
02:04:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.559520 14587 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.561337 14587 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 6 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.562624 14528 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 6 candidate_status { last_received { term: 5 index: 16 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.563143 14528 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 4 FOLLOWER]: Advancing to term 6
02:04:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:26.564584 14086 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.571084 14528 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 6.
02:04:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.572244 14084 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:26.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.572988 14587 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 6 FOLLOWER]: Leader election won for term 6
02:04:26.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.574028 14587 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 6 LEADER]: Becoming Leader. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Running, Role: LEADER
02:04:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.575024 14587 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:26.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:26.585119 13425 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 reported cstate change: term changed from 5 to 6, leader changed from 7f3c1e9950ff41b09847d354de219256 (127.12.234.67) to 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66). New cstate: current_term: 6 leader_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: HEALTHY } } }
02:04:26.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:26.998169 14086 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111) [suppressed 2 similar messages]
02:04:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:27.001041 14086 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:27.166446 14528 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 6 FOLLOWER]: Refusing update from remote peer 473a4c326e6c4936ad3fcff3ade5d4d9: Log matching property violated. Preceding OpId in replica: term: 4 index: 12. Preceding OpId from leader: term: 6 index: 17. (index mismatch)
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:27.168130 14587 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.001s
02:04:28.447 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:28.447 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:28.448 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=3018), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [17ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [19ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [21ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [21ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [36ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [37ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [38ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [39ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [77ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [79ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [81ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [81ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [192ms] refreshing cache from master, [192ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [196ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [197ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [199ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [199ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [633ms] refreshing cache from master, [633ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [637ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [639ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [640ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [640ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [932ms] refreshing cache from master, [932ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [937ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [939ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [941ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [941ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3012ms] refreshing cache from master, [3012ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [3017ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@935262947(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@49b8d997 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408) -> (continuation of Deferred@840060666 after retry RPC@278077284) -> (continuation of Deferred@749571520 after retry RPC@1138930614) -> (continuation of Deferred@151266831 after retry RPC@1605196698), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7938168b -> wakeup thread Time-limited test -> (continuation of Deferred@934015919 after retry RPC@245245855) -> (continuation of Deferred@1782964606 after retry RPC@353931638) -> (continuation of Deferred@1111793335 after retry RPC@327562408) -> (continuation of Deferred@840060666 after retry RPC@278077284) -> (continuation of Deferred@749571520 after retry RPC@1138930614) -> (continuation of Deferred@151266831 after retry RPC@1605196698))) after lookup
02:04:28.449 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:28.450 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] Successfully connected to peer
02:04:28.471 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x1303f685, L:/127.0.0.1:35712 - R:/127.12.234.66:41047] using TOKEN/null
02:04:28.525 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.67:37361
02:04:28.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:28.527195 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:28.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:28.873450 14627 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:28.874011 14627 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:28.874644 14627 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:28.913794 14627 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:28.915081 14627 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.67
02:04:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:28.956082 14627 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:28.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:28.957970 14627 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:28.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:28.959794 14627 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:28.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:28.974136 14633 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:29.416858 14086 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:30.377362 14632 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 14627
02:04:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:30.517460 14627 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.543s	user 0.533s	sys 0.913s
02:04:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:30.518009 14627 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.544s	user 0.533s	sys 0.913s
02:04:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:28.975476 14634 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:30.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:30.520082 14636 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:30.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:30.524673 14635 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1543 milliseconds
02:04:30.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.524755 14627 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.526103 14627 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:30.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.528398 14627 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:30.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.529893 14627 hybrid_clock.cc:648] HybridClock initialized: now 1779069870529854 us; error 50 us; skew 500 ppm
02:04:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.530938 14627 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:30.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.538985 14627 webserver.cc:533] Webserver started at http://127.12.234.67:38593/ using document root <none> and password file <none>
02:04:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.540235 14627 fs_manager.cc:362] Metadata directory not provided
02:04:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.540532 14627 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.549738 14627 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.003s	sys 0.004s
02:04:30.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.554972 14647 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:30.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.556279 14627 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.001s
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.556731 14627 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:03 on dist-test-slave-9xtj"
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.559057 14627 fs_report.cc:389] FS layout report
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/data
02:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:30.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.614527 14627 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.616397 14627 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.617007 14627 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.619856 14627 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.626482 14654 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.635324 14627 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.635670 14627 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
02:04:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.636018 14627 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:30.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.641563 14627 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.641844 14627 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.002s	sys 0.003s
02:04:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.642310 14654 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap starting.
02:04:30.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.711612 14654 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Log is configured to *not* fsync() on all Append() calls
02:04:30.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.779877 14654 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:30.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.781643 14654 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap complete.
02:04:30.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.783385 14654 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent bootstrapping tablet: real 0.142s	user 0.114s	sys 0.021s
02:04:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.804728 14654 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.807668 14654 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Initialized, Role: FOLLOWER
02:04:30.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.809307 14654 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 5.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.816363 14654 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent starting tablet: real 0.033s	user 0.025s	sys 0.008s
02:04:30.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.843967 14627 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.67:37361
02:04:30.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.844286 14762 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.67:37361 every 8 connection(s)
02:04:30.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.846728 14627 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.851011 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 14627
02:04:30.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:30.898 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:30.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.898245 14763 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:30.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.898834 14763 heartbeater.cc:461] Registering TS with master...
02:04:30.900 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.66:41047
02:04:30.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.900290 14763 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:30.900 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

02:04:30.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.901186 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 14061
02:04:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.906412 13426 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.908950 14765 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.909484 14765 heartbeater.cc:461] Registering TS with master...
02:04:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.909565 13426 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.67:39211
02:04:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.910827 14765 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.916625 14766 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.917012 14766 heartbeater.cc:461] Registering TS with master...
02:04:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.916996 13344 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.917832 14766 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:30.921380 13270 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:30.939 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:30.939 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] handling channelInactive
02:04:30.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state READY due to: connection closed
02:04:30.941 [DEBUG - Time-limited test] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:30.942 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.942 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.942 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.966 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:30.967 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:30.967 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [26ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1809481731(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e1e7769 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f1ff55d -> wakeup thread Time-limited test)) after lookup
02:04:30.968 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:30.969 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.969 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.969 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.986 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:30.987 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:30.987 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [26ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [27ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [29ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [29ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1809481731(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e1e7769 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f1ff55d -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149))) after lookup
02:04:30.988 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:30.989 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.989 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:30.990 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.026 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:31.026 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:31.027 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [26ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [27ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [29ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [29ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [47ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [50ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [50ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1809481731(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e1e7769 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f1ff55d -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166))) after lookup
02:04:31.028 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:31.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.029 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.086 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:31.087 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:31.087 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=147), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [26ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [27ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [29ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [29ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [47ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [50ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [50ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [88ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [89ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [89ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [146ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1809481731(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e1e7769 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166) -> (continuation of Deferred@1905605535 after retry RPC@1132130315), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f1ff55d -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166) -> (continuation of Deferred@1905605535 after retry RPC@1132130315))) after lookup
02:04:31.088 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:31.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.090 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.447 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:31.447 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:31.447 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=507), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [26ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [27ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [29ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [29ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [47ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [50ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [50ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [88ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [89ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [89ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [146ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [148ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [149ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [150ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [502ms] refreshing cache from master, [503ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [506ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1809481731(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e1e7769 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166) -> (continuation of Deferred@1905605535 after retry RPC@1132130315) -> (continuation of Deferred@2134572392 after retry RPC@1809096748), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f1ff55d -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166) -> (continuation of Deferred@1905605535 after retry RPC@1132130315) -> (continuation of Deferred@2134572392 after retry RPC@1809096748))) after lookup
02:04:31.449 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:31.450 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.451 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.451 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:31.915129 14763 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.093240 14775 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.093654 14775 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:32.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.096015 14775 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:32.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:32.099788 14460 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:32.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:32.103933 14460 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.109745 14717 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256" is_pre_election: true
02:04:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.110466 14717 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22459bf8d8d44412a52e4c1805d40542 in term 5.
02:04:32.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.111902 14462 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:32.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.112768 14775 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 6 FOLLOWER]: Leader pre-election won for term 7
02:04:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.113066 14775 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.113451 14775 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 6 FOLLOWER]: Advancing to term 7
02:04:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.118359 14775 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.120656 14775 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 7 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.121693 14717 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 7 candidate_status { last_received { term: 6 index: 20 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.122241 14717 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 5 FOLLOWER]: Advancing to term 7
02:04:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:32.124624 14460 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.129873 14717 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22459bf8d8d44412a52e4c1805d40542 in term 7.
02:04:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.130965 14462 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.131774 14775 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 7 FOLLOWER]: Leader election won for term 7
02:04:32.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.132891 14775 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 7 LEADER]: Becoming Leader. State: Replica: 22459bf8d8d44412a52e4c1805d40542, State: Running, Role: LEADER
02:04:32.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.133792 14775 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:32.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.145287 13426 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 reported cstate change: term changed from 6 to 7, leader changed from 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66) to 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65). New cstate: current_term: 7 leader_uuid: "22459bf8d8d44412a52e4c1805d40542" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: UNKNOWN } } }
02:04:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.726153 14717 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 7 FOLLOWER]: Refusing update from remote peer 22459bf8d8d44412a52e4c1805d40542: Log matching property violated. Preceding OpId in replica: term: 5 index: 16. Preceding OpId from leader: term: 7 index: 21. (index mismatch)
02:04:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.727963 14792 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.001s
02:04:32.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:32.730813 14460 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 -> Peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Couldn't send request to peer 473a4c326e6c4936ad3fcff3ade5d4d9. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:32.746757 14796 mvcc.cc:204] Tried to move back new op lower bound from 7287070181281071104 to 7287070173498826752. Current Snapshot: MvccSnapshot[applied={T|T < 7287070181281071104}]
02:04:33.488 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:33.488 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:33.489 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2549), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [22ms] refreshing cache from master, [22ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [26ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [27ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [29ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [29ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [42ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [46ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [47ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [50ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [50ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [82ms] refreshing cache from master, [82ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [86ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [88ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [89ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [89ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [142ms] refreshing cache from master, [142ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [146ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [148ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [149ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [150ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [502ms] refreshing cache from master, [503ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [506ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [509ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [511ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [511ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2542ms] refreshing cache from master, [2543ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [2548ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1809481731(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5e1e7769 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166) -> (continuation of Deferred@1905605535 after retry RPC@1132130315) -> (continuation of Deferred@2134572392 after retry RPC@1809096748) -> (continuation of Deferred@1506841983 after retry RPC@1625370462), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2f1ff55d -> wakeup thread Time-limited test -> (continuation of Deferred@1230064135 after retry RPC@431287149) -> (continuation of Deferred@1330191785 after retry RPC@1897429166) -> (continuation of Deferred@1905605535 after retry RPC@1132130315) -> (continuation of Deferred@2134572392 after retry RPC@1809096748) -> (continuation of Deferred@1506841983 after retry RPC@1625370462))) after lookup
02:04:33.490 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:33.491 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] Successfully connected to peer
02:04:33.510 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x29f16e7d, L:/127.0.0.1:41124 - R:/127.12.234.65:44793] using TOKEN/null
02:04:33.569 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.66:41047
02:04:33.570 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:33.571942 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:04:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:41047
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:04:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:33.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
02:04:33.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:04:33.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:33.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:33.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:33.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:33.921442 14812 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:33.921993 14812 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:33.922652 14812 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:33.958499 14812 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:33.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:33.959362 14812 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.66
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:34.001092 14812 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:41047
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:34.002465 14812 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:34.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:34.004252 14812 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:34.018150 14820 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:35.018875 14460 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 -> Peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Couldn't send request to peer 473a4c326e6c4936ad3fcff3ade5d4d9. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:35.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:34.018576 14821 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:35.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:34.022898 14823 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:35.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:35.397439 14822 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1373 milliseconds
02:04:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.397591 14812 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.400113 14812 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:35.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.403208 14812 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:35.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.404700 14812 hybrid_clock.cc:648] HybridClock initialized: now 1779069875404521 us; error 195 us; skew 500 ppm
02:04:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.405802 14812 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:35.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.413743 14812 webserver.cc:533] Webserver started at http://127.12.234.66:37913/ using document root <none> and password file <none>
02:04:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.415236 14812 fs_manager.cc:362] Metadata directory not provided
02:04:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.415565 14812 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.427340 14812 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.004s	sys 0.002s
02:04:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.434175 14831 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:35.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.435619 14812 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.005s	sys 0.000s
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.436049 14812 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:01 on dist-test-slave-9xtj"
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.438958 14812 fs_report.cc:389] FS layout report
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/data
02:04:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:35.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.494669 14812 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.496657 14812 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.497139 14812 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.499873 14812 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.506532 14838 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.515589 14812 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.515847 14812 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
02:04:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.516065 14812 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.521401 14812 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.521617 14812 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.005s	sys 0.000s
02:04:35.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.522424 14838 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap starting.
02:04:35.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.595475 14838 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Log is configured to *not* fsync() on all Append() calls
02:04:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.710458 14838 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:35.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.711666 14838 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap complete.
02:04:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.713892 14838 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent bootstrapping tablet: real 0.192s	user 0.114s	sys 0.063s
02:04:35.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.741320 14812 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.66:41047
02:04:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.741454 14945 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.66:41047 every 8 connection(s)
02:04:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.745582 14812 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:35.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.748924 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 14812
02:04:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.745167 14838 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.749480 14838 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Initialized, Role: FOLLOWER
02:04:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.751315 14838 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 6.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.764087 14838 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent starting tablet: real 0.050s	user 0.035s	sys 0.008s
02:04:35.779 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:35.798 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.65:44793
02:04:35.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

02:04:35.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.799888 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 14436
02:04:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.807339 14951 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.807835 14951 heartbeater.cc:461] Registering TS with master...
02:04:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.809057 14951 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:35.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.812881 14947 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:35.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.813283 14947 heartbeater.cc:461] Registering TS with master...
02:04:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.814477 14947 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.815991 13271 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.817250 14946 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.817612 14946 heartbeater.cc:461] Registering TS with master...
02:04:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.818189 13346 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:35.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.818611 14946 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:35.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.822039 13425 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:35.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.824810 13425 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.66:38665
02:04:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:35.829063 14946 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:35.837 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] handling channelInactive
02:04:35.837 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state READY due to: connection closed
02:04:35.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:35.837687 13258 connection.cc:582] server connection from 127.12.234.65:51433 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:04:35.838 [DEBUG - Time-limited test] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:35.840 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.840 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.840 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.847 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:35.847 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:35.848 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@738025197(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@428f1f13 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@19b4e8cb -> wakeup thread Time-limited test)) after lookup
02:04:35.849 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:35.850 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.850 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.850 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.867 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:35.867 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:35.867 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [12ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [12ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@738025197(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@428f1f13 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@19b4e8cb -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084))) after lookup
02:04:35.868 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:35.869 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.869 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.869 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.886 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:35.887 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:35.887 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [12ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [12ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [31ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@738025197(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@428f1f13 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@19b4e8cb -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584))) after lookup
02:04:35.888 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:35.889 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.889 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:35.889 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:36.007 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:36.007 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:36.007 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=169), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [12ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [12ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [31ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [50ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [51ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [168ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@738025197(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@428f1f13 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584) -> (continuation of Deferred@206198590 after retry RPC@283928030), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@19b4e8cb -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584) -> (continuation of Deferred@206198590 after retry RPC@283928030))) after lookup
02:04:36.008 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:36.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:36.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:36.009 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:36.187 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:36.187 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:36.188 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=349), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [12ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [12ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [31ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [50ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [51ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [168ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [170ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [171ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [171ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [344ms] refreshing cache from master, [345ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [349ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@738025197(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@428f1f13 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584) -> (continuation of Deferred@206198590 after retry RPC@283928030) -> (continuation of Deferred@616865005 after retry RPC@171902842), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@19b4e8cb -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584) -> (continuation of Deferred@206198590 after retry RPC@283928030) -> (continuation of Deferred@616865005 after retry RPC@171902842))) after lookup
02:04:36.189 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:36.190 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:36.190 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:36.190 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:37.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.102972 14958 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 22459bf8d8d44412a52e4c1805d40542)
02:04:37.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.103451 14958 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.105897 14958 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:37.109323 14649 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:37.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:37.113497 14649 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:37.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.120764 14901 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" is_pre_election: true
02:04:37.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.121670 14901 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 6.
02:04:37.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.123413 14649 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256; no voters: 22459bf8d8d44412a52e4c1805d40542
02:04:37.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.124617 14958 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 7 FOLLOWER]: Leader pre-election won for term 8
02:04:37.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.124888 14958 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 22459bf8d8d44412a52e4c1805d40542)
02:04:37.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.125093 14958 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 7 FOLLOWER]: Advancing to term 8
02:04:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.129786 14958 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.131238 14958 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 8 election: Requested vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.133028 14901 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 8 candidate_status { last_received { term: 7 index: 24 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.133558 14901 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 6 FOLLOWER]: Advancing to term 8
02:04:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:37.134755 14649 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.141887 14901 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 8.
02:04:37.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.143234 14649 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256; no voters: 22459bf8d8d44412a52e4c1805d40542
02:04:37.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.143954 14958 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 8 FOLLOWER]: Leader election won for term 8
02:04:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.145627 14958 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 8 LEADER]: Becoming Leader. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Running, Role: LEADER
02:04:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.146481 14958 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.156286 13425 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 reported cstate change: term changed from 7 to 8, leader changed from 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65) to 7f3c1e9950ff41b09847d354de219256 (127.12.234.67). New cstate: current_term: 8 leader_uuid: "7f3c1e9950ff41b09847d354de219256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: UNKNOWN } } }
02:04:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.559962 14901 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 8 FOLLOWER]: Refusing update from remote peer 7f3c1e9950ff41b09847d354de219256: Log matching property violated. Preceding OpId in replica: term: 6 index: 20. Preceding OpId from leader: term: 8 index: 25. (index mismatch)
02:04:37.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.562043 14958 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Connected to new peer: Peer: permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
02:04:37.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:37.594363 14649 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 -> Peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Couldn't send request to peer 22459bf8d8d44412a52e4c1805d40542. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:37.687 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:37.688 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:37.688 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1850), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [8ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [12ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [12ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [31ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [31ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [50ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [51ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [51ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [164ms] refreshing cache from master, [165ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [168ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [170ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [171ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [171ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [344ms] refreshing cache from master, [345ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [349ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [350ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [352ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [352ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [1844ms] refreshing cache from master, [1845ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1849ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@738025197(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@428f1f13 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584) -> (continuation of Deferred@206198590 after retry RPC@283928030) -> (continuation of Deferred@616865005 after retry RPC@171902842) -> (continuation of Deferred@1931834257 after retry RPC@30649944), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@19b4e8cb -> wakeup thread Time-limited test -> (continuation of Deferred@1316234057 after retry RPC@1882745084) -> (continuation of Deferred@121241187 after retry RPC@880089584) -> (continuation of Deferred@206198590 after retry RPC@283928030) -> (continuation of Deferred@616865005 after retry RPC@171902842) -> (continuation of Deferred@1931834257 after retry RPC@30649944))) after lookup
02:04:37.689 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:37.692 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] Successfully connected to peer
02:04:37.719 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x80fb3907, L:/127.0.0.1:41386 - R:/127.12.234.67:37361] using TOKEN/null
02:04:37.773 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.65:44793
02:04:37.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:37.775542 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:44793
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43241
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:38.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:38.117370 14987 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:38.117832 14987 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:38.118302 14987 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:38.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:38.154047 14987 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:38.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:38.155182 14987 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.65
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:38.199608 14987 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:44793
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43241
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:38.201319 14987 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:38.203231 14987 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:38.216858 14996 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:39.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:38.218750 14997 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:39.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:39.628340 14999 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:39.632965 14998 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1414 milliseconds
02:04:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.633065 14987 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.634512 14987 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.636922 14987 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:39.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.638402 14987 hybrid_clock.cc:648] HybridClock initialized: now 1779069879638415 us; error 92 us; skew 500 ppm
02:04:39.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.639318 14987 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:39.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.647153 14987 webserver.cc:533] Webserver started at http://127.12.234.65:43241/ using document root <none> and password file <none>
02:04:39.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.648524 14987 fs_manager.cc:362] Metadata directory not provided
02:04:39.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.648823 14987 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.657547 14987 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.003s	sys 0.005s
02:04:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.663182 15012 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.664810 14987 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.000s
02:04:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.665182 14987 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:59 on dist-test-slave-9xtj"
02:04:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.667303 14987 fs_report.cc:389] FS layout report
02:04:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/data
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:39.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.727450 14987 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.729342 14987 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.729841 14987 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:39.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.732976 14987 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:39.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.740412 15019 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.753527 14987 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.753857 14987 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.015s	user 0.002s	sys 0.001s
02:04:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.754112 14987 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:39.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.760593 14987 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:39.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.760874 14987 ts_tablet_manager.cc:595] Time spent register tablets: real 0.007s	user 0.007s	sys 0.000s
02:04:39.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.761410 15019 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap starting.
02:04:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.856748 15019 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Log is configured to *not* fsync() on all Append() calls
02:04:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.959770 15019 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:39.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.960862 15019 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap complete.
02:04:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.962550 15019 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent bootstrapping tablet: real 0.202s	user 0.133s	sys 0.061s
02:04:39.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.966308 14987 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.65:44793
02:04:39.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.966807 15126 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.65:44793 every 8 connection(s)
02:04:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.969069 14987 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:39.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:39.973999 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 14987
02:04:39.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:40.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.000115 15019 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:40.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.005069 15019 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22459bf8d8d44412a52e4c1805d40542, State: Initialized, Role: FOLLOWER
02:04:40.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.007045 15019 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 7.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:40.043 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:40.044 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.67:37361
02:04:40.044 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:40.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.044826 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 14627
02:04:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.060654 15019 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent starting tablet: real 0.098s	user 0.041s	sys 0.035s
02:04:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.066385 15127 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.066905 15127 heartbeater.cc:461] Registering TS with master...
02:04:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.068403 15127 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.069132 15128 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.069669 15128 heartbeater.cc:461] Registering TS with master...
02:04:40.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.071022 15128 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.073972 13425 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:40.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.076361 13346 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.077836 13425 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.65:36851
02:04:40.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.083675 15127 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:40.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:40.093541 15137 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.013338 (+     0us) reactor.cc:678] Submitting negotiation task for server connection from 127.12.234.67:39957 (local address 127.12.234.65:44793)
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.056477 (+ 43139us) server_negotiation.cc:207] Beginning negotiation
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.056531 (+    54us) server_negotiation.cc:400] Waiting for connection header
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.056628 (+    97us) server_negotiation.cc:408] Connection header received
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.056936 (+   308us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.056991 (+    55us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.057587 (+   596us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.057887 (+   300us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:40.091345 (+ 33458us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.12.234.67:39957: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:04:40.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":42001,"thread_start_us":41444,"threads_started":1}
02:04:40.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:40.095 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] handling channelInactive
02:04:40.095 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state READY due to: connection closed
02:04:40.097 [DEBUG - Time-limited test] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:40.098 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.098 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.098 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.104964 15132 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:40.107 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:40.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.105347 15132 heartbeater.cc:461] Registering TS with master...
02:04:40.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.105954 15132 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:40.108 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:40.108 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [11ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@169580585(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@6dc8c726 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7840afcb -> wakeup thread Time-limited test)) after lookup
02:04:40.109 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:40.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:40.110186 13270 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:40.110 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.111 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.111 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.127 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:40.127 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:40.127 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [11ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [12ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [15ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [15ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@169580585(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@6dc8c726 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7840afcb -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797))) after lookup
02:04:40.128 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:40.129 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.129 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.129 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.167 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:40.167 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:40.167 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [11ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [12ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [15ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [15ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@169580585(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@6dc8c726 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7840afcb -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827))) after lookup
02:04:40.169 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:40.170 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.170 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.175 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.266 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:40.267 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:40.267 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=171), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [11ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [12ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [15ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [15ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [78ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [79ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [166ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [170ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@169580585(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@6dc8c726 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827) -> (continuation of Deferred@271880677 after retry RPC@1259380217), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7840afcb -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827) -> (continuation of Deferred@271880677 after retry RPC@1259380217))) after lookup
02:04:40.268 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:40.269 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.269 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.269 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.586 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:40.586 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:40.586 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=491), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [11ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [12ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [15ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [15ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [78ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [79ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [166ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [170ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [171ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [173ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [173ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [489ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@169580585(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@6dc8c726 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827) -> (continuation of Deferred@271880677 after retry RPC@1259380217) -> (continuation of Deferred@2105139841 after retry RPC@843667330), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7840afcb -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827) -> (continuation of Deferred@271880677 after retry RPC@1259380217) -> (continuation of Deferred@2105139841 after retry RPC@843667330))) after lookup
02:04:40.587 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:40.588 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:40.589 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:41.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.261777 15141 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:41.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.262334 15141 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:41.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.264868 15141 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:41.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:41.268270 14835 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:41.272332 14835 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:41.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.277884 15082 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542" is_pre_election: true
02:04:41.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.278651 15082 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 7.
02:04:41.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.280741 14833 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:41.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.281692 15141 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 8 FOLLOWER]: Leader pre-election won for term 9
02:04:41.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.282030 15141 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:41.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.282256 15141 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 8 FOLLOWER]: Advancing to term 9
02:04:41.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.287042 15141 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:41.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.288656 15141 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 9 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:41.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.289845 15082 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 9 candidate_status { last_received { term: 8 index: 28 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:41.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.290359 15082 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 7 FOLLOWER]: Advancing to term 9
02:04:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:41.292111 14835 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:41.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.297422 15082 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 9.
02:04:41.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.298444 14833 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:41.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.299263 15141 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 9 FOLLOWER]: Leader election won for term 9
02:04:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.300976 15141 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 9 LEADER]: Becoming Leader. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Running, Role: LEADER
02:04:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.301911 15141 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.312418 13425 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 reported cstate change: term changed from 8 to 9, leader changed from 7f3c1e9950ff41b09847d354de219256 (127.12.234.67) to 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66). New cstate: current_term: 9 leader_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: HEALTHY } } }
02:04:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:41.728943 14835 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:41.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.805976 15082 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 9 FOLLOWER]: Refusing update from remote peer 473a4c326e6c4936ad3fcff3ade5d4d9: Log matching property violated. Preceding OpId in replica: term: 7 index: 24. Preceding OpId from leader: term: 9 index: 29. (index mismatch)
02:04:41.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:41.807766 15141 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
02:04:42.066 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:42.067 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:42.067 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1971), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [2ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [3ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [11ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [12ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [15ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [15ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [30ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [31ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [33ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [33ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [66ms] refreshing cache from master, [67ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [70ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [72ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [78ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [79ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [166ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [170ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [171ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [173ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [173ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [489ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [491ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [493ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [493ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [1966ms] refreshing cache from master, [1966ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1970ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@169580585(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@6dc8c726 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827) -> (continuation of Deferred@271880677 after retry RPC@1259380217) -> (continuation of Deferred@2105139841 after retry RPC@843667330) -> (continuation of Deferred@61133569 after retry RPC@2066309552), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7840afcb -> wakeup thread Time-limited test -> (continuation of Deferred@1376713720 after retry RPC@793585797) -> (continuation of Deferred@1023092593 after retry RPC@2050238827) -> (continuation of Deferred@271880677 after retry RPC@1259380217) -> (continuation of Deferred@2105139841 after retry RPC@843667330) -> (continuation of Deferred@61133569 after retry RPC@2066309552))) after lookup
02:04:42.068 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:42.069 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] Successfully connected to peer
02:04:42.093 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc499c0d7, L:/127.0.0.1:46298 - R:/127.12.234.66:41047] using TOKEN/null
02:04:42.143 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.67:37361
02:04:42.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:42.145889 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:42.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:42.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:42.516036 15169 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:42.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:42.516646 15169 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:42.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:42.517247 15169 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:42.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:42.552876 15169 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:42.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:42.553750 15169 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.67
02:04:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:42.594417 15169 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:42.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:42.596218 15169 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:42.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:42.597988 15169 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:42.612630 15175 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:44.034818 14835 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:44.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:44.014938 15174 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 15169
02:04:44.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:42.613986 15176 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:44.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:44.069396 15169 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.455s	user 0.502s	sys 0.939s
02:04:44.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:44.069792 15169 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.456s	user 0.502s	sys 0.939s
02:04:44.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:44.074496 15177 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:04:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:44.075961 15178 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.076067 15169 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.077687 15169 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.079980 15169 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.081423 15169 hybrid_clock.cc:648] HybridClock initialized: now 1779069884081404 us; error 83 us; skew 500 ppm
02:04:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.082422 15169 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:44.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.089272 15169 webserver.cc:533] Webserver started at http://127.12.234.67:38593/ using document root <none> and password file <none>
02:04:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.090348 15169 fs_manager.cc:362] Metadata directory not provided
02:04:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.090605 15169 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.098855 15169 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.001s
02:04:44.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.104097 15190 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:44.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.105400 15169 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
02:04:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.105827 15169 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:03 on dist-test-slave-9xtj"
02:04:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.108278 15169 fs_report.cc:389] FS layout report
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/data
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:44.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.175942 15169 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.177877 15169 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.178467 15169 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:44.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.181360 15169 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:44.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.187767 15197 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:44.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.197281 15169 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:44.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.197541 15169 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.001s	sys 0.001s
02:04:44.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.197780 15169 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.203385 15169 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.203681 15169 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.003s	sys 0.000s
02:04:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.204282 15197 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap starting.
02:04:44.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.288466 15197 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Log is configured to *not* fsync() on all Append() calls
02:04:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.428224 15197 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:44.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.429343 15197 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap complete.
02:04:44.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.431279 15197 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent bootstrapping tablet: real 0.228s	user 0.178s	sys 0.037s
02:04:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.440100 15169 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.67:37361
02:04:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.440450 15304 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.67:37361 every 8 connection(s)
02:04:44.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.444644 15169 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:44.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.445612 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 15169
02:04:44.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:44.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.456542 15197 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:44.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.461323 15197 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Initialized, Role: FOLLOWER
02:04:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.463179 15197 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 8.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:44.517 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:44.519 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.66:41047
02:04:44.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

02:04:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.520147 15197 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent starting tablet: real 0.088s	user 0.059s	sys 0.027s
02:04:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.520715 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 14812
02:04:44.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.561277 15306 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:44.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.561844 15306 heartbeater.cc:461] Registering TS with master...
02:04:44.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.563192 15306 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.568431 13344 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:44.573 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] handling channelInactive
02:04:44.574 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state READY due to: connection closed
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:44.575133 15311 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.486301 (+     0us) reactor.cc:678] Submitting negotiation task for server connection from 127.12.234.66:54609 (local address 127.12.234.67:37361)
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.519863 (+ 33562us) server_negotiation.cc:207] Beginning negotiation
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.519896 (+    33us) server_negotiation.cc:400] Waiting for connection header
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.519970 (+    74us) server_negotiation.cc:408] Connection header received
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.520236 (+   266us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.520272 (+    36us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.520678 (+   406us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.520944 (+   266us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0518 02:04:44.573171 (+ 52227us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.12.234.66:54609: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:04:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":32852,"thread_start_us":30395,"threads_started":1}
02:04:44.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:44.583 [DEBUG - Time-limited test] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:44.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.584509 15305 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:44.585 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.584920 15305 heartbeater.cc:461] Registering TS with master...
02:04:44.585 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.585 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.585897 15305 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:44.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.587880 15312 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:44.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.588392 15312 heartbeater.cc:461] Registering TS with master...
02:04:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.589288 15312 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.589759 13426 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.592552 13270 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.593122 13426 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.67:43669
02:04:44.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:44.596629 15305 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:44.606 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:44.606 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:44.607 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [23ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1619458734(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5518f4ca -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@af4f4b3 -> wakeup thread Time-limited test)) after lookup
02:04:44.607 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:44.609 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.609 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.609 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.627 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:44.627 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:44.627 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [23ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [24ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [26ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [26ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [43ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1619458734(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5518f4ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@af4f4b3 -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610))) after lookup
02:04:44.628 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:44.629 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.630 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.630 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.667 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:44.668 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:44.668 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=85), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [23ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [24ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [26ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [26ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [43ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [45ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [47ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [47ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [83ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1619458734(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5518f4ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@af4f4b3 -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276))) after lookup
02:04:44.669 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:44.670 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.670 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.670 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.706 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:44.706 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:44.706 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=123), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [23ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [24ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [26ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [26ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [43ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [45ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [47ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [47ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [83ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [85ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [87ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [87ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [122ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1619458734(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5518f4ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276) -> (continuation of Deferred@2146945908 after retry RPC@654855289), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@af4f4b3 -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276) -> (continuation of Deferred@2146945908 after retry RPC@654855289))) after lookup
02:04:44.707 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:44.708 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.708 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.708 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.947 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:44.947 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:44.947 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=364), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [23ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [24ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [26ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [26ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [43ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [45ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [47ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [47ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [83ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [85ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [87ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [87ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [122ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [124ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [125ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [125ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [359ms] refreshing cache from master, [359ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [363ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1619458734(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5518f4ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276) -> (continuation of Deferred@2146945908 after retry RPC@654855289) -> (continuation of Deferred@599541172 after retry RPC@1186119230), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@af4f4b3 -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276) -> (continuation of Deferred@2146945908 after retry RPC@654855289) -> (continuation of Deferred@599541172 after retry RPC@1186119230))) after lookup
02:04:44.948 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:44.950 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.950 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:44.950 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047
02:04:45.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.711727 15318 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:45.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.712127 15318 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.714520 15318 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:45.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:45.719650 15014 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:45.724563 15014 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:45.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.728960 15260 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256" is_pre_election: true
02:04:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.729804 15260 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22459bf8d8d44412a52e4c1805d40542 in term 8.
02:04:45.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.731242 15016 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:45.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.732079 15318 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 9 FOLLOWER]: Leader pre-election won for term 10
02:04:45.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.732533 15318 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 473a4c326e6c4936ad3fcff3ade5d4d9)
02:04:45.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.732767 15318 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 9 FOLLOWER]: Advancing to term 10
02:04:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.737655 15318 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.739238 15318 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 10 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:45.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.740085 15260 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "22459bf8d8d44412a52e4c1805d40542" candidate_term: 10 candidate_status { last_received { term: 9 index: 32 } } ignore_live_leader: false dest_uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:45.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.740973 15260 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 8 FOLLOWER]: Advancing to term 10
02:04:45.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:45.743659 15014 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111)
02:04:45.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.748692 15260 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22459bf8d8d44412a52e4c1805d40542 in term 10.
02:04:45.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.749742 15016 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 7f3c1e9950ff41b09847d354de219256; no voters: 473a4c326e6c4936ad3fcff3ade5d4d9
02:04:45.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.750571 15318 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 10 FOLLOWER]: Leader election won for term 10
02:04:45.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.752393 15318 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 10 LEADER]: Becoming Leader. State: Replica: 22459bf8d8d44412a52e4c1805d40542, State: Running, Role: LEADER
02:04:45.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.753492 15318 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:45.764876 13426 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 reported cstate change: term changed from 9 to 10, leader changed from 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66) to 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65). New cstate: current_term: 10 leader_uuid: "22459bf8d8d44412a52e4c1805d40542" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: UNKNOWN } } }
02:04:46.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:46.284919 15014 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 -> Peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Couldn't send request to peer 473a4c326e6c4936ad3fcff3ade5d4d9. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:46.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:46.302929 15260 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 10 FOLLOWER]: Refusing update from remote peer 22459bf8d8d44412a52e4c1805d40542: Log matching property violated. Preceding OpId in replica: term: 8 index: 28. Preceding OpId from leader: term: 10 index: 33. (index mismatch)
02:04:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:46.304586 15335 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
02:04:46.967 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:46.967 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:46.967 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2384), Traces: [0ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [2ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [23ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [24ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [26ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [26ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [43ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [45ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [47ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [47ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [83ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [85ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [87ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [87ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [119ms] refreshing cache from master, [119ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [122ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [124ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [125ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [125ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [359ms] refreshing cache from master, [359ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [363ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [365ms] sending RPC to server 473a4c326e6c4936ad3fcff3ade5d4d9, [367ms] received response from server 473a4c326e6c4936ad3fcff3ade5d4d9: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [367ms] delaying RPC due to: Network error: Failed to connect to peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047): Connection refused: /127.12.234.66:41047, [2379ms] refreshing cache from master, [2380ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [2383ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1619458734(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@5518f4ca -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276) -> (continuation of Deferred@2146945908 after retry RPC@654855289) -> (continuation of Deferred@599541172 after retry RPC@1186119230) -> (continuation of Deferred@683778133 after retry RPC@2077337661), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@af4f4b3 -> wakeup thread Time-limited test -> (continuation of Deferred@109208685 after retry RPC@729696610) -> (continuation of Deferred@1328916993 after retry RPC@383373276) -> (continuation of Deferred@2146945908 after retry RPC@654855289) -> (continuation of Deferred@599541172 after retry RPC@1186119230) -> (continuation of Deferred@683778133 after retry RPC@2077337661))) after lookup
02:04:46.968 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:46.969 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] Successfully connected to peer
02:04:46.993 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc95e840c, L:/127.0.0.1:35236 - R:/127.12.234.65:44793] using TOKEN/null
02:04:47.045 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.66:41047
02:04:47.045 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:47.047004 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:41047
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:47.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:47.406111 15356 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:47.406879 15356 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:47.407686 15356 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:47.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:47.445674 15356 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:47.447041 15356 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.66
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:47.500475 15356 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.66:41047
02:04:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.66
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.66
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/logs
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:47.501849 15356 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:47.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:47.503886 15356 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:47.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:47.521885 15362 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:48.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:48.720263 15014 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 -> Peer 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047): Couldn't send request to peer 473a4c326e6c4936ad3fcff3ade5d4d9. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.66:41047: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:49.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:48.932700 15361 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 15356
02:04:49.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:49.007550 15361 kernel_stack_watchdog.cc:198] Thread 15356 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
02:04:49.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:49.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:49.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:49.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:49.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:47.523092 15363 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:49.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:49.008585 15356 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.487s	user 0.544s	sys 0.912s
02:04:49.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:49.008989 15356 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.487s	user 0.544s	sys 0.912s
02:04:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:49.011010 15365 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:49.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:49.015136 15364 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1493 milliseconds
02:04:49.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.015142 15356 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:49.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.016626 15356 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:49.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.018713 15356 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.020368 15356 hybrid_clock.cc:648] HybridClock initialized: now 1779069889020170 us; error 142 us; skew 500 ppm
02:04:49.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.021229 15356 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.027731 15356 webserver.cc:533] Webserver started at http://127.12.234.66:37913/ using document root <none> and password file <none>
02:04:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.028867 15356 fs_manager.cc:362] Metadata directory not provided
02:04:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.029088 15356 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:49.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.037488 15356 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.006s
02:04:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.042719 15377 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:49.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.044044 15356 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
02:04:49.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.044482 15356 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:49.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:49.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:01 on dist-test-slave-9xtj"
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.046547 15356 fs_report.cc:389] FS layout report
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/wal
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/data
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:49.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.105448 15356 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:49.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.107205 15356 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.107735 15356 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:49.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.110827 15356 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:49.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.118127 15384 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.127825 15356 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.128083 15356 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.000s	sys 0.003s
02:04:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.128494 15356 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:49.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.133304 15356 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:49.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.133518 15356 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
02:04:49.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.134203 15384 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap starting.
02:04:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.210762 15384 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Log is configured to *not* fsync() on all Append() calls
02:04:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.325841 15384 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.326856 15384 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Bootstrap complete.
02:04:49.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.328766 15384 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent bootstrapping tablet: real 0.195s	user 0.129s	sys 0.062s
02:04:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.354825 15356 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.66:41047
02:04:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.355432 15493 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.66:41047 every 8 connection(s)
02:04:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.358194 15356 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-1/data/info.pb
02:04:49.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.355415 15384 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:49.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.359110 15384 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Initialized, Role: FOLLOWER
02:04:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.360351 15384 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 9.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.366119 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 15356
02:04:49.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:49.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.367864 15384 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9: Time spent starting tablet: real 0.039s	user 0.033s	sys 0.002s
02:04:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.418987 15494 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:49.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.419529 15494 heartbeater.cc:461] Registering TS with master...
02:04:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.420935 15494 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.425334 13425 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.429752 13425 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.66:55351
02:04:49.480 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:49.482 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.65:44793
02:04:49.482 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

02:04:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.483281 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 14987
02:04:49.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.494530 15499 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:49.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.495004 15499 heartbeater.cc:461] Registering TS with master...
02:04:49.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.495826 15499 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:49.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.499854 13344 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:49.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.508496 15500 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:49.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.508908 15500 heartbeater.cc:461] Registering TS with master...
02:04:49.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.509591 15500 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:49.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:49.513584 13271 ts_manager.cc:194] Re-registered known tserver with Master: 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:49.525 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] handling channelInactive
02:04:49.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state READY due to: connection closed
02:04:49.526 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:49.526 [DEBUG - Time-limited test] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:49.527 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.528 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.546 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:49.547 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:49.547 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [20ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1968498865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3322666 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96f3ea -> wakeup thread Time-limited test)) after lookup
02:04:49.548 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:49.549 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.549 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.549 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.566 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:49.566 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:49.566 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [20ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [21ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [23ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [23ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [39ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1968498865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3322666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96f3ea -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946))) after lookup
02:04:49.567 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:49.569 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.569 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.569 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.587 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:49.587 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:49.587 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=61), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [20ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [21ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [23ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [23ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [39ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [41ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [43ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [43ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [60ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1968498865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3322666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96f3ea -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128))) after lookup
02:04:49.588 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:49.589 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.589 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.589 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.686 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:49.686 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:49.687 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=160), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [20ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [21ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [23ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [23ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [39ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [41ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [43ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [43ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [60ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [62ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [63ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [63ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [160ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1968498865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3322666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128) -> (continuation of Deferred@1161465385 after retry RPC@1807553900), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96f3ea -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128) -> (continuation of Deferred@1161465385 after retry RPC@1807553900))) after lookup
02:04:49.688 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:49.689 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.689 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.689 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.887 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:49.887 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)[L],473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:49.888 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=361), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [20ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [21ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [23ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [23ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [39ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [41ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [43ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [43ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [60ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [62ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [63ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [63ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [160ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [161ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [163ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [163ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [356ms] refreshing cache from master, [356ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [361ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1968498865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3322666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128) -> (continuation of Deferred@1161465385 after retry RPC@1807553900) -> (continuation of Deferred@270085468 after retry RPC@319577158), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96f3ea -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128) -> (continuation of Deferred@1161465385 after retry RPC@1807553900) -> (continuation of Deferred@270085468 after retry RPC@319577158))) after lookup
02:04:49.889 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] connecting to peer
02:04:49.890 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.891 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] cleaning up while in state CONNECTING due to: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:49.891 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793
02:04:50.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.434852 15494 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:50.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.596518 15505 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 22459bf8d8d44412a52e4c1805d40542)
02:04:50.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.596937 15505 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.599778 15505 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:50.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:50.603253 15192 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:50.607447 15192 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.614178 15447 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" is_pre_election: true
02:04:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.615127 15447 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 9.
02:04:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.616771 15192 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256; no voters: 22459bf8d8d44412a52e4c1805d40542
02:04:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.617587 15505 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 10 FOLLOWER]: Leader pre-election won for term 11
02:04:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.617910 15505 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 22459bf8d8d44412a52e4c1805d40542)
02:04:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.618155 15505 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 10 FOLLOWER]: Advancing to term 11
02:04:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.623829 15505 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.625737 15505 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 11 election: Requested vote from peers 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793), 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66:41047)
02:04:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.627372 15447 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "7f3c1e9950ff41b09847d354de219256" candidate_term: 11 candidate_status { last_received { term: 10 index: 36 } } ignore_live_leader: false dest_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9"
02:04:50.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.628002 15447 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 9 FOLLOWER]: Advancing to term 11
02:04:50.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:50.629719 15192 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111)
02:04:50.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.635715 15447 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f3c1e9950ff41b09847d354de219256 in term 11.
02:04:50.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.637153 15192 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 473a4c326e6c4936ad3fcff3ade5d4d9, 7f3c1e9950ff41b09847d354de219256; no voters: 22459bf8d8d44412a52e4c1805d40542
02:04:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.638120 15505 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 11 FOLLOWER]: Leader election won for term 11
02:04:50.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.640482 15505 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 11 LEADER]: Becoming Leader. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Running, Role: LEADER
02:04:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.641706 15505 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.650494 13425 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 reported cstate change: term changed from 10 to 11, leader changed from 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65) to 7f3c1e9950ff41b09847d354de219256 (127.12.234.67). New cstate: current_term: 11 leader_uuid: "7f3c1e9950ff41b09847d354de219256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: UNKNOWN } } }
02:04:50.746 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:50.747 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:50.747 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1220), Traces: [0ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [2ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [2ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [20ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [21ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [23ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [23ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [36ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [39ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [41ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [43ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [43ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [56ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [60ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [62ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [63ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [63ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [160ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [161ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [163ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [163ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [356ms] refreshing cache from master, [356ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [361ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [362ms] sending RPC to server 22459bf8d8d44412a52e4c1805d40542, [365ms] received response from server 22459bf8d8d44412a52e4c1805d40542: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [365ms] delaying RPC due to: Network error: Failed to connect to peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793): Connection refused: /127.12.234.65:44793, [1216ms] refreshing cache from master, [1216ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [1220ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@1968498865(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3322666 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128) -> (continuation of Deferred@1161465385 after retry RPC@1807553900) -> (continuation of Deferred@270085468 after retry RPC@319577158) -> (continuation of Deferred@2117138609 after retry RPC@1794713476), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c96f3ea -> wakeup thread Time-limited test -> (continuation of Deferred@2015316712 after retry RPC@2130786946) -> (continuation of Deferred@56133019 after retry RPC@1979415128) -> (continuation of Deferred@1161465385 after retry RPC@1807553900) -> (continuation of Deferred@270085468 after retry RPC@319577158) -> (continuation of Deferred@2117138609 after retry RPC@1794713476))) after lookup
02:04:50.748 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:50.750 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] Successfully connected to peer
02:04:50.787 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xf3ede159, L:/127.0.0.1:58806 - R:/127.12.234.67:37361] using TOKEN/null
02:04:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.799001 15447 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 11 FOLLOWER]: Refusing update from remote peer 7f3c1e9950ff41b09847d354de219256: Log matching property violated. Preceding OpId in replica: term: 9 index: 32. Preceding OpId from leader: term: 11 index: 38. (index mismatch)
02:04:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.801103 15510 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [LEADER]: Connected to new peer: Peer: permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.001s
02:04:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:50.801993 15192 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 -> Peer 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793): Couldn't send request to peer 22459bf8d8d44412a52e4c1805d40542. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.65:44793: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:50.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.817917 15526 mvcc.cc:204] Tried to move back new op lower bound from 7287070257147813888 to 7287070252054642688. Current Snapshot: MvccSnapshot[applied={T|T < 7287070257147813888}]
02:04:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.827476 15524 mvcc.cc:204] Tried to move back new op lower bound from 7287070272694689792 to 7287070272076263424. Current Snapshot: MvccSnapshot[applied={T|T < 7287070272694689792}]
02:04:50.884 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.65:44793
02:04:50.885 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:50.886622 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:44793
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43241
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:51.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:51.234429 15533 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:51.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:51.235190 15533 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:51.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:51.235975 15533 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:51.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:51.274156 15533 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:51.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:51.275025 15533 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.65
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:51.315038 15533 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.65:44793
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.65
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43241
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.65
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/logs
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:51.316403 15533 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:51.318089 15533 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:51.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:51.332942 15540 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:52.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:51.335325 15541 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:52.792912 15543 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:52.795796 15542 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1457 milliseconds
02:04:52.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.795941 15533 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.797232 15533 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:52.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.799423 15533 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:52.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.800879 15533 hybrid_clock.cc:648] HybridClock initialized: now 1779069892800838 us; error 43 us; skew 500 ppm
02:04:52.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.801967 15533 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:52.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.808867 15533 webserver.cc:533] Webserver started at http://127.12.234.65:43241/ using document root <none> and password file <none>
02:04:52.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.809913 15533 fs_manager.cc:362] Metadata directory not provided
02:04:52.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.810180 15533 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.819078 15533 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.000s
02:04:52.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.824352 15554 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:52.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.825603 15533 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
02:04:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.826040 15533 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:03:59 on dist-test-slave-9xtj"
02:04:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.828145 15533 fs_report.cc:389] FS layout report
02:04:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/wal
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/data
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.903482 15533 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:52.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.905179 15533 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:52.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.905695 15533 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:52.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.908736 15533 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:52.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.915202 15562 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.924736 15533 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.925146 15533 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.002s	sys 0.001s
02:04:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.925506 15533 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.932934 15533 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.933156 15533 ts_tablet_manager.cc:595] Time spent register tablets: real 0.008s	user 0.008s	sys 0.000s
02:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:52.933856 15562 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap starting.
02:04:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.014357 15562 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Log is configured to *not* fsync() on all Append() calls
02:04:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.129307 15562 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.130338 15562 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Bootstrap complete.
02:04:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.131844 15562 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent bootstrapping tablet: real 0.199s	user 0.127s	sys 0.065s
02:04:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.137198 15533 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.65:44793
02:04:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.137383 15669 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.65:44793 every 8 connection(s)
02:04:53.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.140439 15533 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-0/data/info.pb
02:04:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.143756 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 15533
02:04:53.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.155956 15562 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.161643 15562 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22459bf8d8d44412a52e4c1805d40542, State: Initialized, Role: FOLLOWER
02:04:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.163086 15562 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 10.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:53.188 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:53.190 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.12.234.67:37361
02:04:53.190 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:53.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.190814 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 15169
02:04:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.189090 15562 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542: Time spent starting tablet: real 0.057s	user 0.043s	sys 0.013s
02:04:53.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.214694 15670 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.215448 15670 heartbeater.cc:461] Registering TS with master...
02:04:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.216920 15670 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.220554 15671 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.221006 15671 heartbeater.cc:461] Registering TS with master...
02:04:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.222096 15671 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:53.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.222877 13425 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.226334 15673 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.226773 15673 heartbeater.cc:461] Registering TS with master...
02:04:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.226603 13344 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.226866 13425 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.65:34351
02:04:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.227807 15673 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.232038 13271 ts_manager.cc:194] Re-registered known tserver with Master: 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:53.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:53.233750 15670 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:53.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:53.238 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] handling channelInactive
02:04:53.239 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state READY due to: connection closed
02:04:53.239 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: connection closed
02:04:53.247 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:53.247 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:53.248 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [1ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@514341914(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3d4eed4c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@665eccc1 -> wakeup thread Time-limited test)) after lookup
02:04:53.248 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:53.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.251 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.267 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:53.267 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:53.268 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [1ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [13ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [13ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@514341914(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3d4eed4c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@665eccc1 -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229))) after lookup
02:04:53.268 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:53.270 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.270 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.270 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.286 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:53.287 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:53.287 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [1ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [13ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [13ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [32ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@514341914(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3d4eed4c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@665eccc1 -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757))) after lookup
02:04:53.288 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:53.289 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.290 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.290 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.427 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:53.427 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:53.427 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=189), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [1ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [13ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [13ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [32ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [50ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [52ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [52ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [188ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@514341914(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3d4eed4c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757) -> (continuation of Deferred@297571589 after retry RPC@914610578), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@665eccc1 -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757) -> (continuation of Deferred@297571589 after retry RPC@914610578))) after lookup
02:04:53.428 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:53.430 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.430 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.430 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.906 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:53.907 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047),7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)[L]]}]
02:04:53.907 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=669), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [1ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [13ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [13ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [32ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [50ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [52ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [52ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [188ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [190ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [192ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [192ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [664ms] refreshing cache from master, [664ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [668ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@514341914(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3d4eed4c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757) -> (continuation of Deferred@297571589 after retry RPC@914610578) -> (continuation of Deferred@1602506140 after retry RPC@2002993335), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@665eccc1 -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757) -> (continuation of Deferred@297571589 after retry RPC@914610578) -> (continuation of Deferred@1602506140 after retry RPC@2002993335))) after lookup
02:04:53.908 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] connecting to peer
02:04:53.909 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.909 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] cleaning up while in state CONNECTING due to: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:53.909 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361) for tablet 1aab78234d3348cabb4f11d3aee773bc: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361
02:04:54.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.391326 15682 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 11 FOLLOWER]: Starting pre-election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:54.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.391814 15682 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 11 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:54.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.394308 15682 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 12 pre-election: Requested pre-vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:54.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:54.398679 15381 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:54.404735 15381 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 12 pre-election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.410578 15625 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542" is_pre_election: true
02:04:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.411585 15625 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 10.
02:04:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.413576 15379 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 12 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.414641 15682 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 11 FOLLOWER]: Leader pre-election won for term 12
02:04:54.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.414958 15682 raft_consensus.cc:493] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 11 FOLLOWER]: Starting leader election (detected failure of leader 7f3c1e9950ff41b09847d354de219256)
02:04:54.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.415325 15682 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 11 FOLLOWER]: Advancing to term 12
02:04:54.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:54.418756 13457 debug-util.cc:398] Leaking SignalData structure 0x7b08000a93c0 after lost signal to thread 13388
02:04:54.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:54.419505 13457 debug-util.cc:398] Leaking SignalData structure 0x7b08000a8760 after lost signal to thread 13460
02:04:54.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.420326 15682 raft_consensus.cc:515] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 12 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:54.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.422235 15682 leader_election.cc:290] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 12 election: Requested vote from peers 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361), 22459bf8d8d44412a52e4c1805d40542 (127.12.234.65:44793)
02:04:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.424773 15625 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "1aab78234d3348cabb4f11d3aee773bc" candidate_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" candidate_term: 12 candidate_status { last_received { term: 11 index: 40 } } ignore_live_leader: false dest_uuid: "22459bf8d8d44412a52e4c1805d40542"
02:04:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.425307 15625 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 10 FOLLOWER]: Advancing to term 12
02:04:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:54.427495 15381 leader_election.cc:336] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 12 election: RPC error from VoteRequest() call to peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111)
02:04:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.432905 15625 raft_consensus.cc:2468] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 12 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 473a4c326e6c4936ad3fcff3ade5d4d9 in term 12.
02:04:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.434059 15379 leader_election.cc:304] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [CANDIDATE]: Term 12 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 22459bf8d8d44412a52e4c1805d40542, 473a4c326e6c4936ad3fcff3ade5d4d9; no voters: 7f3c1e9950ff41b09847d354de219256
02:04:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.434777 15682 raft_consensus.cc:2804] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 12 FOLLOWER]: Leader election won for term 12
02:04:54.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.437186 15682 raft_consensus.cc:697] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [term 12 LEADER]: Becoming Leader. State: Replica: 473a4c326e6c4936ad3fcff3ade5d4d9, State: Running, Role: LEADER
02:04:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.439314 15682 consensus_queue.cc:237] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 12, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:54.452975 13425 catalog_manager.cc:5696] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 reported cstate change: term changed from 11 to 12, leader changed from 7f3c1e9950ff41b09847d354de219256 (127.12.234.67) to 473a4c326e6c4936ad3fcff3ade5d4d9 (127.12.234.66). New cstate: current_term: 12 leader_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } health_report { overall_health: HEALTHY } } }
02:04:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:54.938997 15381 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:55.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:55.033181 15625 raft_consensus.cc:1275] T 1aab78234d3348cabb4f11d3aee773bc P 22459bf8d8d44412a52e4c1805d40542 [term 12 FOLLOWER]: Refusing update from remote peer 473a4c326e6c4936ad3fcff3ade5d4d9: Log matching property violated. Preceding OpId in replica: term: 10 index: 36. Preceding OpId from leader: term: 12 index: 41. (index mismatch)
02:04:55.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:55.034806 15682 consensus_queue.cc:1048] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
02:04:55.747 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 1aab78234d3348cabb4f11d3aee773bc for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523' with partition [<start>, <end>)
02:04:55.748 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=1aab78234d3348cabb4f11d3aee773bc@[22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793),473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)[L],7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)]}]
02:04:55.748 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=1aab78234d3348cabb4f11d3aee773bc, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2510), Traces: [0ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [1ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: connection closed, [1ms] delaying RPC due to: Network error: connection closed, [4ms] refreshing cache from master, [4ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [9ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [10ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [13ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [13ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [24ms] refreshing cache from master, [24ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [28ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [30ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [32ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [32ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [44ms] refreshing cache from master, [44ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [48ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [50ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [52ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [52ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [184ms] refreshing cache from master, [184ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [188ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [190ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [192ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [192ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [664ms] refreshing cache from master, [664ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [668ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, [669ms] sending RPC to server 7f3c1e9950ff41b09847d354de219256, [671ms] received response from server 7f3c1e9950ff41b09847d354de219256: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [671ms] delaying RPC due to: Network error: Failed to connect to peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361): Connection refused: /127.12.234.67:37361, [2504ms] refreshing cache from master, [2504ms] Sub RPC GetTableLocations: sending RPC to server master-127.12.234.124:38173, [2509ms] Sub RPC GetTableLocations: received response from server master-127.12.234.124:38173: OK, deferred=Deferred@514341914(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$239/0x00007f53303371b0@3d4eed4c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757) -> (continuation of Deferred@297571589 after retry RPC@914610578) -> (continuation of Deferred@1602506140 after retry RPC@2002993335) -> (continuation of Deferred@349238957 after retry RPC@2036038264), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@665eccc1 -> wakeup thread Time-limited test -> (continuation of Deferred@837322413 after retry RPC@557870229) -> (continuation of Deferred@337125585 after retry RPC@1223596757) -> (continuation of Deferred@297571589 after retry RPC@914610578) -> (continuation of Deferred@1602506140 after retry RPC@2002993335) -> (continuation of Deferred@349238957 after retry RPC@2036038264))) after lookup
02:04:55.749 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] connecting to peer
02:04:55.750 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] Successfully connected to peer
02:04:55.823 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x1b464334, L:/127.0.0.1:57280 - R:/127.12.234.66:41047] using TOKEN/null
02:04:55.890 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.12.234.67:37361
02:04:55.891 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:55.892844 13225 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu
02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:56.292729 15716 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:56.293236 15716 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:56.293700 15716 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:56.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:56.330711 15716 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:56.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:56.332103 15716 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.234.67
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:56.395650 15716 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.234.84:41411
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.234.67:37361
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.234.67
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38593
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.234.126:34017,127.12.234.125:39317,127.12.234.124:38173
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.234.67
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/logs
02:04:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 01:43:23 UTC on e7f111948823
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11951
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:56.397574 15716 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:56.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:56.399834 15716 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:56.436409 15723 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:57.221802 15381 consensus_peers.cc:597] T 1aab78234d3348cabb4f11d3aee773bc P 473a4c326e6c4936ad3fcff3ade5d4d9 -> Peer 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361): Couldn't send request to peer 7f3c1e9950ff41b09847d354de219256. Status: Network error: Client connection negotiation failed: client connection to 127.12.234.67:37361: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:04:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:57.839176 15721 debug-util.cc:398] Leaking SignalData structure 0x7b08000271c0 after lost signal to thread 15716
02:04:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:57.909360 15721 kernel_stack_watchdog.cc:198] Thread 15716 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
02:04:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:56.437428 15722 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:57.910633 15716 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.473s	user 0.493s	sys 0.898s
02:04:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:57.911019 15716 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.473s	user 0.493s	sys 0.898s
02:04:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:57.911350 15724 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1473 milliseconds
02:04:57.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:57.912586 15726 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:57.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.912609 15716 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:57.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.915882 15716 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:57.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.918116 15716 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.919515 15716 hybrid_clock.cc:648] HybridClock initialized: now 1779069897919470 us; error 62 us; skew 500 ppm
02:04:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.920495 15716 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:57.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.927500 15716 webserver.cc:533] Webserver started at http://127.12.234.67:38593/ using document root <none> and password file <none>
02:04:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.928656 15716 fs_manager.cc:362] Metadata directory not provided
02:04:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.928942 15716 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.937867 15716 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.008s	sys 0.000s
02:04:57.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.943298 15736 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
02:04:57.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.944736 15716 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
02:04:57.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.945175 15716 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data,/tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:57.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f3c1e9950ff41b09847d354de219256"
02:04:57.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 02:04:03 on dist-test-slave-9xtj"
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:57.947381 15716 fs_report.cc:389] FS layout report
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/wal
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/data
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:58.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.009809 15716 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
02:04:58.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.011544 15716 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.012038 15716 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:58.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.015323 15716 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:58.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.022811 15743 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
02:04:58.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.031613 15716 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
02:04:58.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.031875 15716 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
02:04:58.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.032091 15716 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
02:04:58.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.037313 15716 ts_tablet_manager.cc:616] Registered 1 tablets
02:04:58.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.037545 15716 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.002s
02:04:58.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.038146 15743 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap starting.
02:04:58.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.112144 15743 log.cc:826] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Log is configured to *not* fsync() on all Append() calls
02:04:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.254690 15716 rpc_server.cc:307] RPC server started. Bound to: 127.12.234.67:37361
02:04:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.254766 15850 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.234.67:37361 every 8 connection(s)
02:04:58.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.258642 15716 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskFcNZ7a/test-tmp/mini-kudu-cluster6527106432948422456/ts-2/data/info.pb
02:04:58.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.261157 13225 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu as pid 15716
02:04:58.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
02:04:58.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.324554 15743 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:58.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.326057 15743 tablet_bootstrap.cc:492] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Bootstrap complete.
02:04:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.328289 15743 ts_tablet_manager.cc:1403] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent bootstrapping tablet: real 0.291s	user 0.212s	sys 0.069s
02:04:58.337 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.349512 15743 raft_consensus.cc:359] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 11 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.353166 15743 raft_consensus.cc:740] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 11 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f3c1e9950ff41b09847d354de219256, State: Initialized, Role: FOLLOWER
02:04:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.354321 15743 consensus_queue.cc:260] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 11.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7f3c1e9950ff41b09847d354de219256" member_type: VOTER last_known_addr { host: "127.12.234.67" port: 37361 } } peers { permanent_uuid: "22459bf8d8d44412a52e4c1805d40542" member_type: VOTER last_known_addr { host: "127.12.234.65" port: 44793 } } peers { permanent_uuid: "473a4c326e6c4936ad3fcff3ade5d4d9" member_type: VOTER last_known_addr { host: "127.12.234.66" port: 41047 } }
02:04:58.360 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779069844523, tablet=1aab78234d3348cabb4f11d3aee773bc, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
02:04:58.366 [DEBUG - main] (Connection.java:575) [peer master-127.12.234.126:34017(127.12.234.126:34017)] disconnecting while in state READY
02:04:58.367 [DEBUG - main] (Connection.java:575) [peer master-127.12.234.124:38173(127.12.234.124:38173)] disconnecting while in state READY
02:04:58.367 [DEBUG - main] (Connection.java:575) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] disconnecting while in state READY
02:04:58.370 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] handling channelInactive
02:04:58.370 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.234.126:34017(127.12.234.126:34017)] handling channelInactive
02:04:58.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.234.126:34017(127.12.234.126:34017)] cleaning up while in state READY due to: connection closed
02:04:58.371 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.12.234.124:38173(127.12.234.124:38173)] handling channelInactive
02:04:58.372 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.12.234.124:38173(127.12.234.124:38173)] cleaning up while in state READY due to: connection closed
02:04:58.373 [DEBUG - main] (Connection.java:575) [peer 22459bf8d8d44412a52e4c1805d40542(127.12.234.65:44793)] disconnecting while in state TERMINATED
02:04:58.373 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 473a4c326e6c4936ad3fcff3ade5d4d9(127.12.234.66:41047)] cleaning up while in state READY due to: connection closed
02:04:58.374 [DEBUG - main] (Connection.java:575) [peer 7f3c1e9950ff41b09847d354de219256(127.12.234.67:37361)] disconnecting while in state TERMINATED
02:04:58.374 [DEBUG - main] (Connection.java:575) [peer master-127.12.234.125:39317(127.12.234.125:39317)] disconnecting while in state READY
02:04:58.375 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:04:58.377 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.234.125:39317(127.12.234.125:39317)] handling channelInactive
02:04:58.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.234.125:39317(127.12.234.125:39317)] cleaning up while in state READY due to: connection closed
02:04:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.381721 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 15533
02:04:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.388430 15852 heartbeater.cc:344] Connected to a master server at 127.12.234.125:39317
02:04:58.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.388970 15852 heartbeater.cc:461] Registering TS with master...
02:04:58.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.390445 15852 heartbeater.cc:507] Master 127.12.234.125:39317 requested a full tablet report, sending...
02:04:58.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.395921 13344 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.403247 15743 ts_tablet_manager.cc:1434] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256: Time spent starting tablet: real 0.074s	user 0.018s	sys 0.009s
02:04:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.408595 15851 heartbeater.cc:344] Connected to a master server at 127.12.234.124:38173
02:04:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.408984 15851 heartbeater.cc:461] Registering TS with master...
02:04:58.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.409930 15851 heartbeater.cc:507] Master 127.12.234.124:38173 requested a full tablet report, sending...
02:04:58.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.412858 15806 raft_consensus.cc:3060] T 1aab78234d3348cabb4f11d3aee773bc P 7f3c1e9950ff41b09847d354de219256 [term 11 FOLLOWER]: Advancing to term 12
02:04:58.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.413208 13426 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:58.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.416093 13426 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.234.67:59479
02:04:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.421264 15851 heartbeater.cc:499] Master 127.12.234.124:38173 was elected leader, sending a full tablet report...
02:04:58.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.434537 15853 heartbeater.cc:344] Connected to a master server at 127.12.234.126:34017
02:04:58.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.435065 15853 heartbeater.cc:461] Registering TS with master...
02:04:58.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.436110 15853 heartbeater.cc:507] Master 127.12.234.126:34017 requested a full tablet report, sending...
02:04:58.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.508591 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 15356
02:04:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.535187 13270 ts_manager.cc:194] Re-registered known tserver with Master: 7f3c1e9950ff41b09847d354de219256 (127.12.234.67:37361)
02:04:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 02:04:58.554661 15739 connection.cc:582] server connection from 127.12.234.66:52391 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:04:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.555078 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 15716
02:04:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.583556 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 13238
02:04:58.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.619504 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 13308
02:04:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 02:04:58.654350 13225 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskFcNZ7a/build/tsan/bin/kudu with pid 13383
02:04:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T02:04:59Z chronyd exiting

Time: 74.409

OK (1 test)

