JUnit version 4.13.2
.23:51:28.004 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783468287995
23:51:28.077 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
23:51:28.118 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:51:28.133 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskRjNV2q/test-tmp
23:51:28.156 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:51:28.160 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskRjNV2q/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:51:28.363 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

23:51:28.539 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:51:28.541 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

23:51:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:28.545130 30575 mini_oidc.cc:150] Starting JWKS server
23:51:28.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:28.548739 30575 webserver.cc:534] Webserver started at http://127.0.0.1:46409/ using document root <none> and password file <none>
23:51:28.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:28.549263 30575 mini_oidc.cc:186] Starting OIDC Discovery server
23:51:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:28.551707 30575 webserver.cc:534] Webserver started at http://127.0.0.1:35941/ using document root <none> and password file <none>
23:51:28.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:51:28Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
23:51:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:51:28Z Disabled control of system clock
23:51:28.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:28.613127 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:28.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:28.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/wal
23:51:28.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data
23:51:28.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:28.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:28.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:28.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:28.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:28.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:28.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/logs
23:51:28.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data/info.pb
23:51:28.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:28.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.254:45193
23:51:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.254
23:51:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json with env {}
23:51:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.926281 30587 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.926803 30587 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.927248 30587 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:28.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.959653 30587 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:51:28.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.960105 30587 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.960420 30587 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.960752 30587 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:51:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.961119 30587 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:28.961354 30587 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46409/jwks.json
23:51:28.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:28.996980 30587 master_runner.cc:387] Master server non-default flags:
23:51:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/wal
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.254:45193
23:51:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json
23:51:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data/info.pb
23:51:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.254
23:51:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/logs
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:29.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:28.998538 30587 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:29.000464 30587 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:29.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:29.013437 30593 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:29.014374 30594 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.287303 30595 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1271 milliseconds
23:51:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.289201 30596 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:30.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.292476 30587 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.279s	user 0.382s	sys 0.889s
23:51:30.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.292781 30587 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.279s	user 0.382s	sys 0.889s
23:51:30.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.293054 30587 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:30.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.294520 30587 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.297171 30587 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:30.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.298525 30587 hybrid_clock.cc:648] HybridClock initialized: now 1783468290298494 us; error 38 us; skew 500 ppm
23:51:30.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.299484 30587 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:30.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.306712 30587 webserver.cc:534] Webserver started at http://127.29.219.254:41585/ using document root <none> and password file <none>
23:51:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.307893 30587 fs_manager.cc:362] Metadata directory not provided
23:51:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.308138 30587 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.308859 30587 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:30.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.313396 30587 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data/instance:
23:51:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d04e8e41fbe144c4a3fb042dd44bcd13"
23:51:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:30 on dist-test-slave-hdrm"
23:51:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.314781 30587 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/wal/instance:
23:51:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d04e8e41fbe144c4a3fb042dd44bcd13"
23:51:30.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:30 on dist-test-slave-hdrm"
23:51:30.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.322851 30587 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.001s
23:51:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.329053 30603 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.330869 30587 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.002s
23:51:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.331233 30587 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/wal
23:51:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d04e8e41fbe144c4a3fb042dd44bcd13"
23:51:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:30 on dist-test-slave-hdrm"
23:51:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.331677 30587 fs_report.cc:389] FS layout report
23:51:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/wal
23:51:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/wal
23:51:30.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data/data
23:51:30.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:30.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:30.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.401422 30587 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.402817 30587 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.403199 30587 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.475960 30587 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.254:45193
23:51:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.476017 30655 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.254:45193 every 8 connection(s)
23:51:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.478935 30587 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/data/info.pb
23:51:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.482396 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 30587
23:51:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.482890 30575 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-0/wal/instance
23:51:30.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.486404 30656 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.488135 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/wal
23:51:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data
23:51:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/logs
23:51:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data/info.pb
23:51:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.253:42131
23:51:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.253
23:51:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json with env {}
23:51:30.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.499701 30656 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } has no permanent_uuid. Determining permanent_uuid...
23:51:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.530795 30656 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } has no permanent_uuid. Determining permanent_uuid...
23:51:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.533640 30606 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:30.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.538305 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.587881 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 1
23:51:30.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.593292 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.661736 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 2
23:51:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.666778 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:30.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.758306 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 3
23:51:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.762377 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.804445 30658 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.804981 30658 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.805425 30658 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.837664 30658 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:51:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.837927 30658 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.838179 30658 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.838372 30658 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:51:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.838551 30658 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.838687 30658 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46409/jwks.json
23:51:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.875272 30658 master_runner.cc:387] Master server non-default flags:
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/wal
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.253:42131
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data/info.pb
23:51:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.253
23:51:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/logs
23:51:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.876753 30658 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.878777 30658 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.889796 30667 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:30.905822 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 4
23:51:30.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.914465 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:31.214030 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 5
23:51:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:31.223456 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:31.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:31.771004 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 6
23:51:31.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:31.794771 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.294363 30666 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 30658
23:51:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.706125 30658 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.815s	user 0.624s	sys 1.183s
23:51:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.706538 30658 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.815s	user 0.624s	sys 1.183s
23:51:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:30.892015 30668 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.709062 30670 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.712191 30669 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1816 milliseconds
23:51:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.712242 30658 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.713477 30658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:32.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.715979 30658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.717377 30658 hybrid_clock.cc:648] HybridClock initialized: now 1783468292717342 us; error 36 us; skew 500 ppm
23:51:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.718264 30658 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.724987 30658 webserver.cc:534] Webserver started at http://127.29.219.253:44417/ using document root <none> and password file <none>
23:51:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.726011 30658 fs_manager.cc:362] Metadata directory not provided
23:51:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.726264 30658 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:32.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.726830 30658 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:32.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.731151 30658 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data/instance:
23:51:32.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1288cfa9538463ba34383f847a88eb3"
23:51:32.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:32 on dist-test-slave-hdrm"
23:51:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.732707 30658 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/wal/instance:
23:51:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1288cfa9538463ba34383f847a88eb3"
23:51:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:32 on dist-test-slave-hdrm"
23:51:32.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.740214 30658 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.002s
23:51:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.745985 30679 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.746994 30658 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
23:51:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.747351 30658 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/wal
23:51:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1288cfa9538463ba34383f847a88eb3"
23:51:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:32 on dist-test-slave-hdrm"
23:51:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.747730 30658 fs_report.cc:389] FS layout report
23:51:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/wal
23:51:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/wal
23:51:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data/data
23:51:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:32.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.800031 30658 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:32.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.801625 30658 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:32.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.802160 30658 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:32.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.855356 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 7
23:51:32.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.863611 30656 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.253:42131: Network error: Client connection negotiation failed: client connection to 127.29.219.253:42131: connect: Connection refused (error 111)
23:51:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.876431 30658 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.253:42131
23:51:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.876521 30732 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.253:42131 every 8 connection(s)
23:51:32.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.879288 30658 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/data/info.pb
23:51:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.885504 30733 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.888299 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 30658
23:51:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.888625 30575 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-1/wal/instance
23:51:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.890664 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/wal
23:51:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data
23:51:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/logs
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data/info.pb
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.252:38747
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.252
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json with env {}
23:51:32.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.898262 30733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } has no permanent_uuid. Determining permanent_uuid...
23:51:32.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.922691 30733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } has no permanent_uuid. Determining permanent_uuid...
23:51:32.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.937681 30733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } has no permanent_uuid. Determining permanent_uuid...
23:51:32.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.940301 30683 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.945089 30733 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.252:38747: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:32.994683 30733 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } attempt: 1
23:51:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:32.999095 30733 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.252:38747: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:33.067559 30733 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } attempt: 2
23:51:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.071887 30733 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.252:38747: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:33.163367 30733 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } attempt: 3
23:51:33.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.168843 30733 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.252:38747: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.202193 30735 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.202732 30735 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.203156 30735 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.238365 30735 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
23:51:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.238642 30735 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.238837 30735 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.239020 30735 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
23:51:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.239199 30735 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.239343 30735 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46409/jwks.json
23:51:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:33.274377 30735 master_runner.cc:387] Master server non-default flags:
23:51:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data
23:51:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/wal
23:51:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.252:38747
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json
23:51:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data/info.pb
23:51:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.252
23:51:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/logs
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:33.275697 30735 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:33.277401 30735 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.288841 30745 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:33.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:33.312232 30733 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } attempt: 4
23:51:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.316917 30733 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.252:38747: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:33.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:33.616536 30733 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } attempt: 5
23:51:33.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.623541 30733 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.252:38747: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:34.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.171156 30733 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } attempt: 6
23:51:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:34.187521 30733 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.219.252:38747: Network error: Client connection negotiation failed: client connection to 127.29.219.252:38747: connect: Connection refused (error 111)
23:51:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:33.290875 30746 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:34.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:34.522686 30748 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:34.526018 30747 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1231 milliseconds
23:51:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.526199 30735 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.527504 30735 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:34.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.530076 30735 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.531699 30735 hybrid_clock.cc:648] HybridClock initialized: now 1783468294531650 us; error 48 us; skew 500 ppm
23:51:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.532645 30735 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.538800 30735 webserver.cc:534] Webserver started at http://127.29.219.252:39585/ using document root <none> and password file <none>
23:51:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.539772 30735 fs_manager.cc:362] Metadata directory not provided
23:51:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.540069 30735 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.540652 30735 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:34.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.545120 30735 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data/instance:
23:51:34.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8045498754784ff5aee9d7282a2d5157"
23:51:34.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:34 on dist-test-slave-hdrm"
23:51:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.546339 30735 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/wal/instance:
23:51:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8045498754784ff5aee9d7282a2d5157"
23:51:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:34 on dist-test-slave-hdrm"
23:51:34.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.553351 30735 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.000s	sys 0.008s
23:51:34.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.558969 30757 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.560030 30735 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:51:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.560411 30735 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/wal
23:51:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8045498754784ff5aee9d7282a2d5157"
23:51:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:34 on dist-test-slave-hdrm"
23:51:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.560786 30735 fs_report.cc:389] FS layout report
23:51:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/wal
23:51:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/wal
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data/data
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:34.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:34.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:34.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:34.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.621860 30735 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:34.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.623530 30735 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:34.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.624053 30735 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.697711 30735 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.252:38747
23:51:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.697806 30809 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.252:38747 every 8 connection(s)
23:51:34.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.700695 30735 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/data/info.pb
23:51:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.703652 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 30735
23:51:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.704161 30575 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/master-2/wal/instance
23:51:34.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.707897 30810 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.724332 30810 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } has no permanent_uuid. Determining permanent_uuid...
23:51:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.752084 30810 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } has no permanent_uuid. Determining permanent_uuid...
23:51:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.762141 30810 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } has no permanent_uuid. Determining permanent_uuid...
23:51:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.778439 30810 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: Bootstrap starting.
23:51:34.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.785458 30810 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: Neither blocks nor log segments found. Creating new log.
23:51:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.787197 30810 log.cc:826] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: Log is configured to *not* fsync() on all Append() calls
23:51:34.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.791401 30810 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: No bootstrap required, opened a new log
23:51:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.811113 30810 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.811709 30810 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.812029 30810 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8045498754784ff5aee9d7282a2d5157, State: Initialized, Role: FOLLOWER
23:51:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.812701 30810 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.816511 30819 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.817273 30810 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [sys.catalog]: configured and running, proceeding with master startup.
23:51:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.817322 30819 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [sys.catalog]: This master's current role is: FOLLOWER
23:51:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:34.843472 30830 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:34.843837 30830 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.838843 30735 master_runner.cc:187] Error getting master registration for 127.29.219.254:45193: OK, instance_id { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" instance_seqno: 1783468290455902 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:51:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.858865 30735 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:51:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:34.879563 30763 tablet.cc:2406] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:34.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.954128 30656 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } attempt: 8
23:51:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.966794 30656 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } has no permanent_uuid. Determining permanent_uuid...
23:51:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.982769 30656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13: Bootstrap starting.
23:51:34.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.989066 30656 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13: Neither blocks nor log segments found. Creating new log.
23:51:34.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.990922 30656 log.cc:826] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13: Log is configured to *not* fsync() on all Append() calls
23:51:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:34.995607 30656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13: No bootstrap required, opened a new log
23:51:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.017360 30656 raft_consensus.cc:359] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.018029 30656 raft_consensus.cc:385] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.018304 30656 raft_consensus.cc:740] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d04e8e41fbe144c4a3fb042dd44bcd13, State: Initialized, Role: FOLLOWER
23:51:35.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.019307 30656 consensus_queue.cc:260] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.022660 30834 sys_catalog.cc:455] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.023892 30834 sys_catalog.cc:458] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: This master's current role is: FOLLOWER
23:51:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.024667 30656 sys_catalog.cc:565] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: configured and running, proceeding with master startup.
23:51:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.060707 30834 raft_consensus.cc:493] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.061255 30834 raft_consensus.cc:515] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.067564 30785 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8045498754784ff5aee9d7282a2d5157" is_pre_election: true
23:51:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.068432 30785 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d04e8e41fbe144c4a3fb042dd44bcd13 in term 0.
23:51:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.070729 30607 leader_election.cc:304] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8045498754784ff5aee9d7282a2d5157, d04e8e41fbe144c4a3fb042dd44bcd13; no voters: 
23:51:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.072198 30834 leader_election.cc:290] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1288cfa9538463ba34383f847a88eb3 (127.29.219.253:42131), 8045498754784ff5aee9d7282a2d5157 (127.29.219.252:38747)
23:51:35.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.072712 30834 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:51:35.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.073015 30834 raft_consensus.cc:493] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:51:35.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.073314 30834 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 0 FOLLOWER]: Advancing to term 1
23:51:35.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.091173 30834 raft_consensus.cc:515] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.094215 30845 catalog_manager.cc:1593] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.094925 30845 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:35.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.099303 30785 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8045498754784ff5aee9d7282a2d5157"
23:51:35.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.099869 30785 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [term 0 FOLLOWER]: Advancing to term 1
23:51:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.105949 30834 leader_election.cc:290] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [CANDIDATE]: Term 1 election: Requested vote from peers e1288cfa9538463ba34383f847a88eb3 (127.29.219.253:42131), 8045498754784ff5aee9d7282a2d5157 (127.29.219.252:38747)
23:51:35.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.106871 30785 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d04e8e41fbe144c4a3fb042dd44bcd13 in term 1.
23:51:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.062979 30587 master_runner.cc:187] Error getting master registration for 127.29.219.253:42131: OK, instance_id { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" instance_seqno: 1783468292854909 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:51:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.108511 30607 leader_election.cc:304] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8045498754784ff5aee9d7282a2d5157, d04e8e41fbe144c4a3fb042dd44bcd13; no voters: 
23:51:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.109472 30587 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
23:51:35.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.110204 30834 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 1 FOLLOWER]: Leader election won for term 1
23:51:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.110685 30834 raft_consensus.cc:697] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [term 1 LEADER]: Becoming Leader. State: Replica: d04e8e41fbe144c4a3fb042dd44bcd13, State: Running, Role: LEADER
23:51:35.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.111631 30834 consensus_queue.cc:237] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:35.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.126888 30847 sys_catalog.cc:455] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d04e8e41fbe144c4a3fb042dd44bcd13. Latest consensus state: current_term: 1 leader_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.137302 30847 sys_catalog.cc:458] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: This master's current role is: LEADER
23:51:35.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.148047 30849 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
23:51:35.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.155558 30849 catalog_manager.cc:1519] Initializing Kudu cluster ID...
23:51:35.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.164798 30609 tablet.cc:2406] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:35.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.177855 30785 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [term 1 FOLLOWER]: Refusing update from remote peer d04e8e41fbe144c4a3fb042dd44bcd13: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.179622 30834 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.208232 30850 mvcc.cc:204] Tried to move back new op lower bound from 7305086137027350528 to 7305086136804851712. Current Snapshot: MvccSnapshot[applied={T|T < 7305086137027350528}]
23:51:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.215081 30819 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d04e8e41fbe144c4a3fb042dd44bcd13. Latest consensus state: current_term: 1 leader_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.216234 30819 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [sys.catalog]: This master's current role is: FOLLOWER
23:51:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.217191 30847 sys_catalog.cc:455] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.217919 30847 sys_catalog.cc:458] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: This master's current role is: LEADER
23:51:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.226361 30849 catalog_manager.cc:1382] Generated new cluster ID: 9e5b540ced73417fac4a4669250b176d
23:51:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.226738 30849 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
23:51:35.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.230821 30819 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.231487 30819 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157 [sys.catalog]: This master's current role is: FOLLOWER
23:51:35.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.248144 30733 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } attempt: 7
23:51:35.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.278812 30849 catalog_manager.cc:1405] Generated new certificate authority record
23:51:35.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.282562 30849 catalog_manager.cc:1539] Loading token signing keys...
23:51:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.321134 30733 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3: Bootstrap starting.
23:51:35.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.344784 30849 catalog_manager.cc:6069] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13: Generated new TSK 0
23:51:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.345892 30849 catalog_manager.cc:1549] Initializing in-progress tserver states...
23:51:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.356705 30733 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3: Neither blocks nor log segments found. Creating new log.
23:51:35.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.359503 30733 log.cc:826] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3: Log is configured to *not* fsync() on all Append() calls
23:51:35.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.383176 30733 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3: No bootstrap required, opened a new log
23:51:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.383622 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:51:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:35.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:0
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.424737 30733 raft_consensus.cc:359] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:35.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.427666 30733 raft_consensus.cc:385] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:35.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.428432 30733 raft_consensus.cc:740] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e1288cfa9538463ba34383f847a88eb3, State: Initialized, Role: FOLLOWER
23:51:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.430121 30733 consensus_queue.cc:260] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } }
23:51:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.455927 30733 sys_catalog.cc:565] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [sys.catalog]: configured and running, proceeding with master startup.
23:51:35.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.456059 30706 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e1288cfa9538463ba34383f847a88eb3" is_pre_election: true
23:51:35.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.457466 30704 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [term 0 FOLLOWER]: Advancing to term 1
23:51:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.461083 30705 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e1288cfa9538463ba34383f847a88eb3"
23:51:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.470671 30704 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [term 1 FOLLOWER]: Refusing update from remote peer d04e8e41fbe144c4a3fb042dd44bcd13: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.474970 30834 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
23:51:35.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.454205 30858 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.482156 30858 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [sys.catalog]: This master's current role is: FOLLOWER
23:51:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.570992 30858 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d04e8e41fbe144c4a3fb042dd44bcd13. Latest consensus state: current_term: 1 leader_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.571132 30860 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.572304 30860 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [sys.catalog]: This master's current role is: FOLLOWER
23:51:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.572286 30858 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3 [sys.catalog]: This master's current role is: FOLLOWER
23:51:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.608881 30874 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:51:35.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.851562 30830 catalog_manager.cc:1294] Loaded cluster ID: 9e5b540ced73417fac4a4669250b176d
23:51:35.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.852519 30830 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: loading cluster ID for follower catalog manager: success
23:51:35.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.862483 30830 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: acquiring CA information for follower catalog manager: success
23:51:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.870042 30830 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 8045498754784ff5aee9d7282a2d5157: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:51:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.929272 30857 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.929899 30857 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:35.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.930814 30857 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.973949 30834 sys_catalog.cc:455] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d04e8e41fbe144c4a3fb042dd44bcd13" member_type: VOTER last_known_addr { host: "127.29.219.254" port: 45193 } } peers { permanent_uuid: "e1288cfa9538463ba34383f847a88eb3" member_type: VOTER last_known_addr { host: "127.29.219.253" port: 42131 } } peers { permanent_uuid: "8045498754784ff5aee9d7282a2d5157" member_type: VOTER last_known_addr { host: "127.29.219.252" port: 38747 } } }
23:51:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:35.974660 30834 sys_catalog.cc:458] T 00000000000000000000000000000000 P d04e8e41fbe144c4a3fb042dd44bcd13 [sys.catalog]: This master's current role is: LEADER
23:51:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.976475 30857 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:35.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:35.978307 30857 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.193
23:51:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:36.018347 30857 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:0
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:36.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:36.020211 30857 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:36.022480 30857 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:36.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:36.059151 30882 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:36.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:36.631942 30874 catalog_manager.cc:1294] Loaded cluster ID: 9e5b540ced73417fac4a4669250b176d
23:51:36.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:36.632790 30874 catalog_manager.cc:1587] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3: loading cluster ID for follower catalog manager: success
23:51:36.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:36.641656 30874 catalog_manager.cc:1609] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3: acquiring CA information for follower catalog manager: success
23:51:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:36.656875 30874 catalog_manager.cc:1637] T 00000000000000000000000000000000 P e1288cfa9538463ba34383f847a88eb3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:51:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:37.455578 30881 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 30857
23:51:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:36.066989 30883 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:38.355304 30857 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.290s	user 0.573s	sys 1.072s
23:51:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:38.356094 30857 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.291s	user 0.573s	sys 1.072s
23:51:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.363925 30857 server_base.cc:1061] running on GCE node
23:51:38.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:38.365453 30888 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:38.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.366925 30857 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.369551 30857 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.371037 30857 hybrid_clock.cc:648] HybridClock initialized: now 1783468298370930 us; error 49 us; skew 500 ppm
23:51:38.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.372262 30857 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.381768 30857 webserver.cc:534] Webserver started at http://127.29.219.193:43571/ using document root <none> and password file <none>
23:51:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.383435 30857 fs_manager.cc:362] Metadata directory not provided
23:51:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.383772 30857 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.384356 30857 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.391438 30857 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/instance:
23:51:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2e1fdaeeab841af80053f63388320b1"
23:51:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:38 on dist-test-slave-hdrm"
23:51:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.393060 30857 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal/instance:
23:51:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2e1fdaeeab841af80053f63388320b1"
23:51:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:38 on dist-test-slave-hdrm"
23:51:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.403056 30857 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.009s	sys 0.001s
23:51:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.410257 30893 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.411518 30857 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.002s
23:51:38.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.411957 30857 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:38.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2e1fdaeeab841af80053f63388320b1"
23:51:38.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:38 on dist-test-slave-hdrm"
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.412371 30857 fs_report.cc:389] FS layout report
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/data
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.472150 30857 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.474090 30857 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.474663 30857 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:38.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.478181 30857 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:38.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.484125 30857 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:38.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.484396 30857 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:51:38.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.484683 30857 ts_tablet_manager.cc:616] Registered 0 tablets
23:51:38.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.484891 30857 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:51:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.671312 30857 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.193:45855
23:51:38.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.671482 31007 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.193:45855 every 8 connection(s)
23:51:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.674158 30857 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:51:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.677680 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 30857
23:51:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.678287 30575 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal/instance
23:51:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.692819 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:51:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:0
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:38.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:38.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:38.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.745819 31008 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:51:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.746312 31008 heartbeater.cc:461] Registering TS with master...
23:51:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.747565 31008 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:51:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.749486 31009 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:51:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.750138 31009 heartbeater.cc:461] Registering TS with master...
23:51:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.750753 31012 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:51:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.750960 31009 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:51:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.751080 31012 heartbeater.cc:461] Registering TS with master...
23:51:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.752054 31012 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:51:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.754724 30775 ts_manager.cc:194] Registered new tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.755252 30696 ts_manager.cc:194] Registered new tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.757445 30619 ts_manager.cc:194] Registered new tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:38.760118 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.193:47305
23:51:39.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:39.073940 31018 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:39.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:39.074404 31018 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:39.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:39.074826 31018 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:39.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:39.107208 31018 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:39.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:39.108053 31018 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.194
23:51:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:39.144047 31018 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:0
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:39.145345 31018 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:39.147198 31018 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:39.160259 31025 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:39.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:39.764597 31012 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:51:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:40.563066 31024 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 31018
23:51:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:40.605849 31024 kernel_stack_watchdog.cc:198] Thread 31018 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
23:51:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:51:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:51:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:51:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:51:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:39.161442 31026 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:40.606500 31018 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.445s	user 0.509s	sys 0.888s
23:51:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:40.606871 31018 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.445s	user 0.509s	sys 0.888s
23:51:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:40.609922 31028 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:40.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.613569 31018 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:40.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:40.613544 31027 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1450 milliseconds
23:51:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.615131 31018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.619313 31018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.620676 31018 hybrid_clock.cc:648] HybridClock initialized: now 1783468300620625 us; error 48 us; skew 500 ppm
23:51:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.621644 31018 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:40.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.635499 31018 webserver.cc:534] Webserver started at http://127.29.219.194:36759/ using document root <none> and password file <none>
23:51:40.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.636546 31018 fs_manager.cc:362] Metadata directory not provided
23:51:40.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.636853 31018 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:40.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.637563 31018 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.642722 31018 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/instance:
23:51:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451a779f984c42ac861a7b36231ac58d"
23:51:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:40 on dist-test-slave-hdrm"
23:51:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.643926 31018 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal/instance:
23:51:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451a779f984c42ac861a7b36231ac58d"
23:51:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:40 on dist-test-slave-hdrm"
23:51:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.653679 31018 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.004s	sys 0.004s
23:51:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.667891 31035 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.669344 31018 fs_manager.cc:730] Time spent opening block manager: real 0.011s	user 0.003s	sys 0.006s
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.669658 31018 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451a779f984c42ac861a7b36231ac58d"
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:40 on dist-test-slave-hdrm"
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.670024 31018 fs_report.cc:389] FS layout report
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/data
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.724733 31018 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.726269 31018 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.726740 31018 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.729354 31018 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.733511 31018 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:40.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.733749 31018 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:51:40.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.734078 31018 ts_tablet_manager.cc:616] Registered 0 tablets
23:51:40.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.734295 31018 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:51:40.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.888933 31018 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.194:41145
23:51:40.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.889175 31149 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.194:41145 every 8 connection(s)
23:51:40.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.893067 31018 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:51:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.894505 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 31018
23:51:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.894892 30575 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal/instance
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.918668 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:0
23:51:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:51:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:51:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:40.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.946350 31150 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:51:40.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.947021 31150 heartbeater.cc:461] Registering TS with master...
23:51:40.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.948864 31150 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:51:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.954005 30775 ts_manager.cc:194] Registered new tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:51:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.972338 31151 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:51:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.972752 31151 heartbeater.cc:461] Registering TS with master...
23:51:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.973567 31151 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:51:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.976260 30696 ts_manager.cc:194] Registered new tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:51:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.976524 31155 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:51:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.976879 31155 heartbeater.cc:461] Registering TS with master...
23:51:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.977715 31155 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:51:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.981382 30619 ts_manager.cc:194] Registered new tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:51:40.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:40.984292 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.194:50187
23:51:41.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:41.286983 31154 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:41.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:41.287468 31154 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:41.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:41.287942 31154 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:41.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:41.318768 31154 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:41.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:41.319495 31154 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.195
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:41.353611 31154 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:0
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:41.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:51:41.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:51:41.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:41.354964 31154 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:41.357061 31154 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:41.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:41.370533 31165 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:41.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:41.987811 31155 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:51:42.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:42.724387 31168 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:42.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:41.371167 31166 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:42.727941 31154 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.358s	user 0.423s	sys 0.873s
23:51:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:42.728479 31154 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.358s	user 0.423s	sys 0.873s
23:51:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:42.732506 31167 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1359 milliseconds
23:51:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.733006 31154 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.734325 31154 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.737032 31154 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.738458 31154 hybrid_clock.cc:648] HybridClock initialized: now 1783468302738419 us; error 57 us; skew 500 ppm
23:51:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.739485 31154 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.753994 31154 webserver.cc:534] Webserver started at http://127.29.219.195:33295/ using document root <none> and password file <none>
23:51:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.755190 31154 fs_manager.cc:362] Metadata directory not provided
23:51:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.755443 31154 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:42.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.756213 31154 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
23:51:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.761673 31154 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/instance:
23:51:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:51:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:42 on dist-test-slave-hdrm"
23:51:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.762939 31154 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal/instance:
23:51:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:51:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:42 on dist-test-slave-hdrm"
23:51:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.774019 31154 fs_manager.cc:696] Time spent creating directory manager: real 0.010s	user 0.010s	sys 0.002s
23:51:42.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.780308 31175 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:42.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.781591 31154 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
23:51:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.781967 31154 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:51:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:42 on dist-test-slave-hdrm"
23:51:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.782301 31154 fs_report.cc:389] FS layout report
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/data
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:42.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.870358 31154 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:42.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.872164 31154 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:42.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.872673 31154 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:42.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.875643 31154 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:42.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.880082 31154 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:42.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.880324 31154 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
23:51:42.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.880646 31154 ts_tablet_manager.cc:616] Registered 0 tablets
23:51:42.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:42.880852 31154 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
23:51:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.056984 31154 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.195:43891
23:51:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.057102 31288 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.195:43891 every 8 connection(s)
23:51:43.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.059581 31154 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:51:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.068229 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 31154
23:51:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.068724 30575 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal/instance
23:51:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.105109 31289 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:51:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.105620 31289 heartbeater.cc:461] Registering TS with master...
23:51:43.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.106662 31291 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:51:43.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.107015 31291 heartbeater.cc:461] Registering TS with master...
23:51:43.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.106977 31289 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:51:43.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.108373 31291 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:51:43.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.110225 30775 ts_manager.cc:194] Registered new tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:51:43.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.110827 30619 ts_manager.cc:194] Registered new tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:51:43.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.110956 31290 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:51:43.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.111258 31290 heartbeater.cc:461] Registering TS with master...
23:51:43.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.112288 31290 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:51:43.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.112679 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.195:33009
23:51:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.115339 30695 ts_manager.cc:194] Registered new tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:51:43.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:43.134075 30575 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
23:51:43.146 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
23:51:43.147 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

23:51:43.155 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.29.219.254"
      port: 45193
    }
    bound_http_address {
      host: "127.29.219.254"
      port: 41585
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.29.219.253"
      port: 42131
    }
    bound_http_address {
      host: "127.29.219.253"
      port: 44417
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.29.219.252"
      port: 38747
    }
    bound_http_address {
      host: "127.29.219.252"
      port: 39585
    }
  }
}

23:51:43.158 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

23:51:43.161 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.29.219.193"
      port: 45855
    }
    bound_http_address {
      host: "127.29.219.193"
      port: 43571
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.29.219.194"
      port: 41145
    }
    bound_http_address {
      host: "127.29.219.194"
      port: 36759
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.29.219.195"
      port: 43891
    }
    bound_http_address {
      host: "127.29.219.195"
      port: 33295
    }
  }
}

23:51:43.161 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:51:43.330 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
23:51:43.392 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
23:51:43.549 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.219.252:38747(127.29.219.252:38747)] connecting to peer
23:51:43.609 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.219.254:45193(127.29.219.254:45193)] connecting to peer
23:51:43.616 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.219.253:42131(127.29.219.253:42131)] connecting to peer
23:51:43.631 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.29.219.253:42131(127.29.219.253:42131)] Successfully connected to peer
23:51:43.633 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.219.252:38747(127.29.219.252:38747)] Successfully connected to peer
23:51:43.636 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.29.219.254:45193(127.29.219.254:45193)] Successfully connected to peer
23:51:43.735 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.219.254
23:51:43.739 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.219.252
23:51:43.739 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.219.253
23:51:43.973 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xfec5d088, L:/127.0.0.1:57156 - R:/127.29.219.252:38747] using SASL/PLAIN
23:51:43.973 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x23ba76c3, L:/127.0.0.1:51232 - R:/127.29.219.253:42131] using SASL/PLAIN
23:51:43.973 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7eb7c962, L:/127.0.0.1:48988 - R:/127.29.219.254:45193] using SASL/PLAIN
23:51:44.013 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
23:51:44.014 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=504), Traces: [0ms] refreshing cache from master, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.219.252:38747, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.219.254:45193, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.219.253:42131, [474ms] Sub RPC ConnectToMaster: received response from server master-127.29.219.252:38747: OK, [475ms] Sub RPC ConnectToMaster: received response from server master-127.29.219.253:42131: OK, [475ms] Sub RPC ConnectToMaster: received response from server master-127.29.219.254:45193: OK, deferred=null) after lookup
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.060379 30619 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:48988:
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498"
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
23:51:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
23:51:44.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:44.064690 30619 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
23:51:44.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.116590 31291 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:51:44.162 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 6f11a9bf3067442b9a36709c2213d4da
23:51:44.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.176369 31085 tablet_service.cc:1511] Processing CreateTablet for tablet a9a7b9a0d6cf45c78c344887dd124464 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498 [id=6f11a9bf3067442b9a36709c2213d4da]), partition=RANGE (key) PARTITION UNBOUNDED
23:51:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.178493 31085 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a9a7b9a0d6cf45c78c344887dd124464. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:44.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.208523 31223 tablet_service.cc:1511] Processing CreateTablet for tablet a9a7b9a0d6cf45c78c344887dd124464 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498 [id=6f11a9bf3067442b9a36709c2213d4da]), partition=RANGE (key) PARTITION UNBOUNDED
23:51:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.212574 31326 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap starting.
23:51:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.212422 31223 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a9a7b9a0d6cf45c78c344887dd124464. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.215705 30943 tablet_service.cc:1511] Processing CreateTablet for tablet a9a7b9a0d6cf45c78c344887dd124464 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498 [id=6f11a9bf3067442b9a36709c2213d4da]), partition=RANGE (key) PARTITION UNBOUNDED
23:51:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.217592 30943 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a9a7b9a0d6cf45c78c344887dd124464. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:44.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.222918 31326 tablet_bootstrap.cc:654] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Neither blocks nor log segments found. Creating new log.
23:51:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.225481 31326 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Log is configured to *not* fsync() on all Append() calls
23:51:44.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.238559 31326 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: No bootstrap required, opened a new log
23:51:44.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.239363 31326 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent bootstrapping tablet: real 0.027s	user 0.007s	sys 0.014s
23:51:44.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.249778 31328 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap starting.
23:51:44.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.261912 31329 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap starting.
23:51:44.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.262787 31328 tablet_bootstrap.cc:654] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Neither blocks nor log segments found. Creating new log.
23:51:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.265313 31328 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Log is configured to *not* fsync() on all Append() calls
23:51:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.270596 31329 tablet_bootstrap.cc:654] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Neither blocks nor log segments found. Creating new log.
23:51:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.271124 31326 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.272339 31326 raft_consensus.cc:385] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.273604 31329 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Log is configured to *not* fsync() on all Append() calls
23:51:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.272848 31326 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 451a779f984c42ac861a7b36231ac58d, State: Initialized, Role: FOLLOWER
23:51:44.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.275970 31326 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.283336 31328 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: No bootstrap required, opened a new log
23:51:44.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.284243 31328 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent bootstrapping tablet: real 0.035s	user 0.023s	sys 0.007s
23:51:44.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.284394 31326 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent starting tablet: real 0.044s	user 0.021s	sys 0.012s
23:51:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.294065 31329 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: No bootstrap required, opened a new log
23:51:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.294833 31329 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent bootstrapping tablet: real 0.033s	user 0.019s	sys 0.010s
23:51:44.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.337857 31328 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.338616 31328 raft_consensus.cc:385] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.338910 31328 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Initialized, Role: FOLLOWER
23:51:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.340102 31328 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.342135 31329 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.343163 31329 raft_consensus.cc:385] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.343504 31329 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Initialized, Role: FOLLOWER
23:51:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.344641 31329 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.345762 31328 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent starting tablet: real 0.061s	user 0.032s	sys 0.009s
23:51:44.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.350203 31329 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent starting tablet: real 0.055s	user 0.033s	sys 0.019s
23:51:44.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.358001 31334 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:44.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.358615 31334 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.362334 31334 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:51:44.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.391990 31243 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" is_pre_election: true
23:51:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.393179 31243 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 0.
23:51:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.394829 30894 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1; no voters: 
23:51:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.396014 31335 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:51:44.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.396466 31335 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:51:44.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.396824 31335 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 0 FOLLOWER]: Advancing to term 1
23:51:44.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.402880 31105 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d" is_pre_election: true
23:51:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.403573 31105 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 0.
23:51:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.404001 31335 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.406343 31335 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 1 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:51:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.406895 31105 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d"
23:51:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.407445 31105 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 0 FOLLOWER]: Advancing to term 1
23:51:44.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.407477 31243 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:51:44.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.408130 31243 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 0 FOLLOWER]: Advancing to term 1
23:51:44.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.412730 31105 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 1.
23:51:44.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.413828 30897 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1; no voters: 
23:51:44.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.414549 31335 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 1 FOLLOWER]: Leader election won for term 1
23:51:44.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.415118 31335 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 1 LEADER]: Becoming Leader. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Running, Role: LEADER
23:51:44.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.415336 31243 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 1.
23:51:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.416337 31335 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:44.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.427659 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 reported cstate change: term changed from 0 to 1, leader changed from <none> to d2e1fdaeeab841af80053f63388320b1 (127.29.219.193). New cstate: current_term: 1 leader_uuid: "d2e1fdaeeab841af80053f63388320b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: HEALTHY } } }
23:51:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:44.462181 31159 tablet.cc:2406] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:44.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:44.485189 31017 tablet.cc:2406] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:44.504 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 6f11a9bf3067442b9a36709c2213d4da
23:51:44.526 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:44.534 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:44.535 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [11ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=null) after lookup
23:51:44.545 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:44.548 [DEBUG - kudu-nio-3] (Connection.java:757) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] Successfully connected to peer
23:51:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:44.574779 31293 tablet.cc:2406] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:44.619 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x5c6a6025, L:/127.0.0.1:45458 - R:/127.29.219.193:45855] using TOKEN/null
23:51:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.636227 31105 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 1 FOLLOWER]: Refusing update from remote peer d2e1fdaeeab841af80053f63388320b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.636245 31243 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 1 FOLLOWER]: Refusing update from remote peer d2e1fdaeeab841af80053f63388320b1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:44.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.637998 31334 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:44.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.638660 31335 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.673666 31341 mvcc.cc:204] Tried to move back new op lower bound from 7305086175771701248 to 7305086174898028544. Current Snapshot: MvccSnapshot[applied={T|T < 7305086175771701248}]
23:51:44.860 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:51:44.861 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.193:45855
23:51:44.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:51:44.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:44.863742 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 30857
23:51:44.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:44.906 [DEBUG - kudu-nio-3] (Connection.java:239) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] handling channelInactive
23:51:44.907 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state READY due to: connection closed
23:51:44.907 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: connection closed
23:51:44.916 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:44.917 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:44.917 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [13ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: connection closed, [13ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@999204777(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@67193bdb -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ad0a3c2 -> wakeup thread Time-limited test)) after lookup
23:51:44.918 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:44.921 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.921 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.921 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.935 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:44.935 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:44.936 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=41), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [13ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: connection closed, [13ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [23ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [27ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [27ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [41ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@999204777(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@67193bdb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ad0a3c2 -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132))) after lookup
23:51:44.936 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:44.938 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.938 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.939 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.975 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:44.975 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:44.976 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=81), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [13ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: connection closed, [13ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [23ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [27ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [27ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [41ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [42ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [45ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [45ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [76ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@999204777(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@67193bdb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ad0a3c2 -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522))) after lookup
23:51:44.977 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:44.978 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.979 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:44.979 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:45.035 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:45.036 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:45.036 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=141), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [13ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: connection closed, [13ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [23ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [27ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [27ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [41ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [42ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [45ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [45ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [76ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [82ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [85ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [85ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [136ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [141ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@999204777(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@67193bdb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ad0a3c2 -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480))) after lookup
23:51:45.037 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:45.038 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:45.038 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:45.039 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:45.535 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:45.535 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:45.536 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=641), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [13ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: connection closed, [13ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [23ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [27ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [27ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [41ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [42ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [45ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [45ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [76ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [82ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [85ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [85ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [136ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [141ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [142ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [145ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [145ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [636ms] refreshing cache from master, [637ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [641ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@999204777(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@67193bdb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480) -> (continuation of Deferred@1290198300 after retry RPC@546208215), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ad0a3c2 -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480) -> (continuation of Deferred@1290198300 after retry RPC@546208215))) after lookup
23:51:45.537 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:45.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:45.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:45.538 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:46.215 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:46.216 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:46.216 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1322), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [13ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: connection closed, [13ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [23ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [27ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [27ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [41ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [42ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [45ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [45ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [76ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [82ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [85ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [85ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [136ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [141ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [142ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [145ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [145ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [636ms] refreshing cache from master, [637ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [641ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [642ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [644ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [644ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1316ms] refreshing cache from master, [1317ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1321ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@999204777(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@67193bdb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480) -> (continuation of Deferred@1290198300 after retry RPC@546208215) -> (continuation of Deferred@1553895783 after retry RPC@260905954), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ad0a3c2 -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480) -> (continuation of Deferred@1290198300 after retry RPC@546208215) -> (continuation of Deferred@1553895783 after retry RPC@260905954))) after lookup
23:51:46.217 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:46.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:46.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:46.218 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:51:46.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.300369 31354 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:51:46.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.300748 31354 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:46.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.303046 31354 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:46.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.303309 31355 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:51:46.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.304054 31355 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:46.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:46.308460 31036 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:51:46.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.309556 31355 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:46.320122 31176 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:51:46.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.321494 31243 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "451a779f984c42ac861a7b36231ac58d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" is_pre_election: true
23:51:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.322078 31243 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 451a779f984c42ac861a7b36231ac58d in term 1.
23:51:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:46.323335 31036 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:51:46.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.324255 31036 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:51:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.324997 31354 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 1 FOLLOWER]: Leader pre-election won for term 2
23:51:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.325315 31354 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 1 FOLLOWER]: Starting leader election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:51:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.325657 31354 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 1 FOLLOWER]: Advancing to term 2
23:51:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:46.326234 31176 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:51:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.330509 31105 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d" is_pre_election: true
23:51:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.331599 31354 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.332556 31105 raft_consensus.cc:2393] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 0edbfeb3c4ab49e1b0da501a288438f4 in current term 2: Already voted for candidate 451a779f984c42ac861a7b36231ac58d in this term.
23:51:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.333461 31354 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 2 election: Requested vote from peers 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.334075 31243 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "451a779f984c42ac861a7b36231ac58d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:51:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.334614 31243 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 1 FOLLOWER]: Advancing to term 2
23:51:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.336215 31179 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4; no voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1
23:51:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:46.337688 31036 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:51:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.339991 31243 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 451a779f984c42ac861a7b36231ac58d in term 2.
23:51:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.340376 31355 raft_consensus.cc:2749] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
23:51:46.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.340919 31036 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:51:46.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.341578 31354 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 FOLLOWER]: Leader election won for term 2
23:51:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.343186 31354 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 LEADER]: Becoming Leader. State: Replica: 451a779f984c42ac861a7b36231ac58d, State: Running, Role: LEADER
23:51:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.344030 31354 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.353721 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d reported cstate change: term changed from 1 to 2, leader changed from d2e1fdaeeab841af80053f63388320b1 (127.29.219.193) to 451a779f984c42ac861a7b36231ac58d (127.29.219.194). New cstate: current_term: 2 leader_uuid: "451a779f984c42ac861a7b36231ac58d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: UNKNOWN } } }
23:51:46.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.884608 31243 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Refusing update from remote peer 451a779f984c42ac861a7b36231ac58d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
23:51:46.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:46.886210 31377 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
23:51:46.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:46.939769 31036 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:51:49.576 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:49.576 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:49.577 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=4682), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [13ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: connection closed, [13ms] delaying RPC due to: Network error: connection closed, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [22ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [23ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [27ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [27ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [41ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [42ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [45ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [45ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [76ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [82ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [85ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [85ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [136ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [141ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [142ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [145ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [145ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [636ms] refreshing cache from master, [637ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [641ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [642ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [644ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [644ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1316ms] refreshing cache from master, [1317ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1321ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [1323ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1324ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1324ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [4677ms] refreshing cache from master, [4677ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [4682ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@999204777(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@67193bdb -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480) -> (continuation of Deferred@1290198300 after retry RPC@546208215) -> (continuation of Deferred@1553895783 after retry RPC@260905954) -> (continuation of Deferred@681248415 after retry RPC@2141898898), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5ad0a3c2 -> wakeup thread Time-limited test -> (continuation of Deferred@215578838 after retry RPC@1554436132) -> (continuation of Deferred@383858892 after retry RPC@837898522) -> (continuation of Deferred@340909343 after retry RPC@2093040480) -> (continuation of Deferred@1290198300 after retry RPC@546208215) -> (continuation of Deferred@1553895783 after retry RPC@260905954) -> (continuation of Deferred@681248415 after retry RPC@2141898898))) after lookup
23:51:49.578 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:49.579 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] Successfully connected to peer
23:51:49.602 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x94c9de88, L:/127.0.0.1:48214 - R:/127.29.219.194:41145] using TOKEN/null
23:51:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.632661 31036 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:51:49.652 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.193:45855
23:51:49.653 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:49.654996 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.987993 31406 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.988554 31406 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:49.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:49.989208 31406 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:50.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.021394 31406 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:50.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.022455 31406 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.193
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.059569 31406 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.061470 31406 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:50.063217 31406 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.077488 31412 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.085704 31415 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:51.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:50.080705 31413 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:51.330116 31414 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1245 milliseconds
23:51:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.330248 31406 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.331745 31406 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.334721 31406 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.336310 31406 hybrid_clock.cc:648] HybridClock initialized: now 1783468311336272 us; error 41 us; skew 500 ppm
23:51:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.337149 31406 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:51.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.343974 31406 webserver.cc:534] Webserver started at http://127.29.219.193:43571/ using document root <none> and password file <none>
23:51:51.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.344877 31406 fs_manager.cc:362] Metadata directory not provided
23:51:51.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.345140 31406 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.352885 31406 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.003s	sys 0.001s
23:51:51.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.357349 31424 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:51.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.358412 31406 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.004s
23:51:51.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.358760 31406 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:51.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2e1fdaeeab841af80053f63388320b1"
23:51:51.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:38 on dist-test-slave-hdrm"
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.360832 31406 fs_report.cc:389] FS layout report
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/data
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:51.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.453593 31406 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.455195 31406 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.455828 31406 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.458520 31406 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.464862 31431 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:51:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.474083 31406 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:51:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.474331 31406 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
23:51:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.474694 31406 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:51:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.479377 31406 ts_tablet_manager.cc:616] Registered 1 tablets
23:51:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.479769 31406 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.002s
23:51:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.480098 31431 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap starting.
23:51:51.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:51.484730 31036 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111) [suppressed 10 similar messages]
23:51:51.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.540942 31431 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Log is configured to *not* fsync() on all Append() calls
23:51:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.567103 31431 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:51:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.568104 31431 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap complete.
23:51:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.569581 31431 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent bootstrapping tablet: real 0.090s	user 0.056s	sys 0.029s
23:51:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.588948 31431 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.591465 31431 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Initialized, Role: FOLLOWER
23:51:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.593372 31431 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.598591 31431 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent starting tablet: real 0.029s	user 0.024s	sys 0.004s
23:51:51.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.662671 31406 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.193:45855
23:51:51.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.663013 31540 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.193:45855 every 8 connection(s)
23:51:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.665696 31406 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:51:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.666508 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 31406
23:51:51.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:51.714 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:51:51.715 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.194:41145
23:51:51.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:51:51.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.716431 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 31018
23:51:51.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.740084 31541 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:51:51.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.740612 31541 heartbeater.cc:461] Registering TS with master...
23:51:51.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.741163 31542 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:51:51.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.741585 31542 heartbeater.cc:461] Registering TS with master...
23:51:51.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.742043 31541 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:51:51.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.742641 31542 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:51:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.746809 30696 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.747710 30774 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:51.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.751084 31543 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:51:51.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.751431 31543 heartbeater.cc:461] Registering TS with master...
23:51:51.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.752334 31543 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:51:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.756145 30619 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:51.759 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] handling channelInactive
23:51:51.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state READY due to: connection closed
23:51:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:51.760062 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.193:57065
23:51:51.762 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:51.763 [DEBUG - Time-limited test] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:51.764 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.764 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.764 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.775 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:51.775 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:51.776 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1119917222(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6b873a34 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d5dff2f -> wakeup thread Time-limited test)) after lookup
23:51:51.776 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:51.777 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.777 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.777 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.794 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:51.795 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:51.795 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [14ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [15ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [15ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1119917222(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6b873a34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d5dff2f -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821))) after lookup
23:51:51.796 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:51.797 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.797 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.797 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.815 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:51.815 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:51.816 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [14ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [15ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [15ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [33ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [35ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [35ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [53ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1119917222(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6b873a34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d5dff2f -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632))) after lookup
23:51:51.817 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:51.817 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.818 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.818 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.955 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:51.955 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:51.956 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=193), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [14ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [15ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [15ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [33ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [35ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [35ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [53ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [54ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [56ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [56ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [192ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1119917222(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6b873a34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d5dff2f -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655))) after lookup
23:51:51.957 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:51.958 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.959 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:51.959 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:52.155 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:52.155 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:52.156 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=393), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [14ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [15ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [15ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [33ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [35ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [35ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [53ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [54ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [56ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [56ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [192ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [194ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [197ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [197ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [389ms] refreshing cache from master, [389ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [393ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1119917222(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6b873a34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655) -> (continuation of Deferred@162105532 after retry RPC@273146690), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d5dff2f -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655) -> (continuation of Deferred@162105532 after retry RPC@273146690))) after lookup
23:51:52.156 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:52.158 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:52.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:52.158 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.764331 31543 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:51:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.885402 31552 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 451a779f984c42ac861a7b36231ac58d)
23:51:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.885938 31552 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.888594 31552 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.892238 31179 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:51:52.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.897200 31179 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:51:52.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.905422 31496 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "d2e1fdaeeab841af80053f63388320b1" is_pre_election: true
23:51:52.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.906157 31496 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 1.
23:51:52.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.907472 31176 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1; no voters: 451a779f984c42ac861a7b36231ac58d
23:51:52.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.908152 31552 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Leader pre-election won for term 3
23:51:52.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.908474 31552 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 451a779f984c42ac861a7b36231ac58d)
23:51:52.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.908780 31552 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 2 FOLLOWER]: Advancing to term 3
23:51:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.914397 31552 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.915771 31552 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 3 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:51:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.916890 31496 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "d2e1fdaeeab841af80053f63388320b1"
23:51:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.917435 31496 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 1 FOLLOWER]: Advancing to term 3
23:51:52.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:52.919574 31179 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:51:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.924006 31496 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 3.
23:51:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.924791 31176 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1; no voters: 451a779f984c42ac861a7b36231ac58d
23:51:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.925320 31552 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 FOLLOWER]: Leader election won for term 3
23:51:52.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.926581 31552 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 LEADER]: Becoming Leader. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Running, Role: LEADER
23:51:52.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.927649 31552 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:52.938 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:52.937429 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 reported cstate change: term changed from 2 to 3, leader changed from 451a779f984c42ac861a7b36231ac58d (127.29.219.194) to 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195). New cstate: current_term: 3 leader_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: UNKNOWN } } }
23:51:52.938 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:52.939 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1176), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [14ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [15ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [15ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [33ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [35ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [35ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [53ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [54ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [56ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [56ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [192ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [194ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [197ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [197ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [389ms] refreshing cache from master, [389ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [393ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [394ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [396ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [396ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [1169ms] refreshing cache from master, [1169ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1176ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1119917222(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6b873a34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655) -> (continuation of Deferred@162105532 after retry RPC@273146690) -> (continuation of Deferred@282779148 after retry RPC@1068004006), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d5dff2f -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655) -> (continuation of Deferred@162105532 after retry RPC@273146690) -> (continuation of Deferred@282779148 after retry RPC@1068004006))) after lookup
23:51:52.940 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:51:52.941 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:52.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:52.941 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:51:53.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:53.436254 31179 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Couldn't send request to peer 451a779f984c42ac861a7b36231ac58d. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:51:53.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.464120 31496 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 3 FOLLOWER]: Refusing update from remote peer 0edbfeb3c4ab49e1b0da501a288438f4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
23:51:53.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.465409 31552 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
23:51:53.655 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:53.655 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:53.655 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1893), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [13ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [14ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [15ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [15ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [32ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [33ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [35ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [35ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [49ms] refreshing cache from master, [49ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [53ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [54ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [56ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [56ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [189ms] refreshing cache from master, [189ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [192ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [194ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [197ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [197ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [389ms] refreshing cache from master, [389ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [393ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [394ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [396ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [396ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [1169ms] refreshing cache from master, [1169ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1176ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [1177ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [1179ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [1179ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [1889ms] refreshing cache from master, [1889ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1892ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1119917222(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6b873a34 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655) -> (continuation of Deferred@162105532 after retry RPC@273146690) -> (continuation of Deferred@282779148 after retry RPC@1068004006) -> (continuation of Deferred@1816569922 after retry RPC@567153024), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7d5dff2f -> wakeup thread Time-limited test -> (continuation of Deferred@632620299 after retry RPC@931340821) -> (continuation of Deferred@1213128829 after retry RPC@1695695632) -> (continuation of Deferred@1080167736 after retry RPC@933223655) -> (continuation of Deferred@162105532 after retry RPC@273146690) -> (continuation of Deferred@282779148 after retry RPC@1068004006) -> (continuation of Deferred@1816569922 after retry RPC@567153024))) after lookup
23:51:53.656 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:51:53.657 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] Successfully connected to peer
23:51:53.681 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x8a3bd989, L:/127.0.0.1:55072 - R:/127.29.219.195:43891] using TOKEN/null
23:51:53.754 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.194:41145
23:51:53.755 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:51:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:53.756417 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:51:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:41145
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36759
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:54.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.091395 31583 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.091989 31583 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.092643 31583 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.125454 31583 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.126530 31583 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.194
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.161935 31583 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:41145
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:51:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36759
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.163666 31583 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:54.165397 31583 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.178028 31589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.684698 31179 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Couldn't send request to peer 451a779f984c42ac861a7b36231ac58d. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:51:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.582293 31588 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 31583
23:51:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.860684 31588 kernel_stack_watchdog.cc:198] Thread 31583 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 401ms:
23:51:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:51:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:51:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:51:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:51:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:54.178606 31590 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.865787 31583 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.687s	user 0.603s	sys 1.069s
23:51:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.866212 31583 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.688s	user 0.603s	sys 1.069s
23:51:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.866303 31591 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1686 milliseconds
23:51:55.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:55.867811 31592 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:55.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.867882 31583 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:55.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.869109 31583 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:55.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.871199 31583 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:55.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.872573 31583 hybrid_clock.cc:648] HybridClock initialized: now 1783468315872529 us; error 33 us; skew 500 ppm
23:51:55.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.873426 31583 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.879546 31583 webserver.cc:534] Webserver started at http://127.29.219.194:36759/ using document root <none> and password file <none>
23:51:55.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.880563 31583 fs_manager.cc:362] Metadata directory not provided
23:51:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.880820 31583 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.888993 31583 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.001s
23:51:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.893718 31601 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.894816 31583 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
23:51:55.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.895193 31583 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:55.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451a779f984c42ac861a7b36231ac58d"
23:51:55.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:40 on dist-test-slave-hdrm"
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.897361 31583 fs_report.cc:389] FS layout report
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/data
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.974112 31583 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:51:55.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.975719 31583 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:55.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.976173 31583 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:55.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.978960 31583 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:51:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.985379 31609 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:51:55.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.993280 31583 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:51:55.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.993486 31583 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
23:51:55.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.993707 31583 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:51:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.998485 31583 ts_tablet_manager.cc:616] Registered 1 tablets
23:51:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.998734 31583 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
23:51:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:55.999485 31609 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap starting.
23:51:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.051553 31609 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Log is configured to *not* fsync() on all Append() calls
23:51:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.092006 31609 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:51:56.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.092868 31609 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap complete.
23:51:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.094376 31609 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent bootstrapping tablet: real 0.095s	user 0.055s	sys 0.035s
23:51:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.114724 31609 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.117208 31609 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 451a779f984c42ac861a7b36231ac58d, State: Initialized, Role: FOLLOWER
23:51:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.118270 31609 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.122946 31609 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent starting tablet: real 0.028s	user 0.022s	sys 0.007s
23:51:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.176728 31583 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.194:41145
23:51:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.177162 31717 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.194:41145 every 8 connection(s)
23:51:56.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.179160 31583 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:51:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.188230 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 31583
23:51:56.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:56.223 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:51:56.224 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.195:43891
23:51:56.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:51:56.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.226454 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 31154
23:51:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.246050 31719 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:51:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.246577 31719 heartbeater.cc:461] Registering TS with master...
23:51:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.247927 31719 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:51:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.252653 30696 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:51:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.254272 31718 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:51:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.254637 31718 heartbeater.cc:461] Registering TS with master...
23:51:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.255617 31718 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:51:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.259433 30775 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:51:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.261930 31720 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:51:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.262286 31720 heartbeater.cc:461] Registering TS with master...
23:51:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.263175 31720 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:51:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.266016 30619 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:51:56.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:56.268182 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.194:33119
23:51:56.274 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] handling channelInactive
23:51:56.275 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state READY due to: connection closed
23:51:56.275 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:51:56.276 [DEBUG - Time-limited test] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:51:56.277 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.277 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.277 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.294 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:56.295 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:56.295 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [19ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@254827604(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5ea30e23 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e0c7fbb -> wakeup thread Time-limited test)) after lookup
23:51:56.296 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:51:56.297 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.297 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.297 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.314 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:56.314 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:56.314 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [19ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [38ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@254827604(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5ea30e23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e0c7fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097))) after lookup
23:51:56.315 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:51:56.316 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.316 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.316 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.334 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:56.335 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:56.335 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [19ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [38ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [40ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [41ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [41ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [59ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@254827604(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5ea30e23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e0c7fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149))) after lookup
23:51:56.335 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:51:56.336 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.336 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.336 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.435 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:56.435 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:56.435 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=159), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [19ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [38ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [40ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [41ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [41ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [59ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [60ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [61ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [61ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [159ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@254827604(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5ea30e23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149) -> (continuation of Deferred@761183401 after retry RPC@28859825), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e0c7fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149) -> (continuation of Deferred@761183401 after retry RPC@28859825))) after lookup
23:51:56.436 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:51:56.437 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.437 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.437 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.815 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:56.815 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:51:56.815 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=539), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [19ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [38ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [40ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [41ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [41ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [59ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [60ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [61ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [61ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [159ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [160ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [162ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [162ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [536ms] refreshing cache from master, [536ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [539ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@254827604(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5ea30e23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149) -> (continuation of Deferred@761183401 after retry RPC@28859825) -> (continuation of Deferred@1725799358 after retry RPC@625673985), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e0c7fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149) -> (continuation of Deferred@761183401 after retry RPC@28859825) -> (continuation of Deferred@1725799358 after retry RPC@625673985))) after lookup
23:51:56.816 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:51:56.817 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.817 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:56.817 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:51:57.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.271534 31720 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:51:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.401396 31729 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 0edbfeb3c4ab49e1b0da501a288438f4)
23:51:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.401822 31729 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:57.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.404037 31729 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:51:57.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:57.408318 31425 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:51:57.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:57.412652 31425 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:51:57.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.415277 31673 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d" is_pre_election: true
23:51:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.415998 31673 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 2.
23:51:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.417263 31428 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1; no voters: 0edbfeb3c4ab49e1b0da501a288438f4
23:51:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.417963 31729 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 3 FOLLOWER]: Leader pre-election won for term 4
23:51:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.418200 31729 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 0edbfeb3c4ab49e1b0da501a288438f4)
23:51:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.418414 31729 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 3 FOLLOWER]: Advancing to term 4
23:51:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.422924 31729 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:57.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.424350 31729 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 4 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:51:57.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.425149 31673 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d"
23:51:57.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.425624 31673 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 2 FOLLOWER]: Advancing to term 4
23:51:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:57.427711 31425 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:51:57.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.431730 31673 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 4.
23:51:57.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.432581 31428 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1; no voters: 0edbfeb3c4ab49e1b0da501a288438f4
23:51:57.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.433219 31729 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Leader election won for term 4
23:51:57.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.434722 31729 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 LEADER]: Becoming Leader. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Running, Role: LEADER
23:51:57.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.435701 31729 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:51:57.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.445250 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 reported cstate change: term changed from 3 to 4, leader changed from 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195) to d2e1fdaeeab841af80053f63388320b1 (127.29.219.193). New cstate: current_term: 4 leader_uuid: "d2e1fdaeeab841af80053f63388320b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: HEALTHY } } }
23:51:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:57.842175 31425 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 -> Peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Couldn't send request to peer 0edbfeb3c4ab49e1b0da501a288438f4. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:51:57.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.895635 31673 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Refusing update from remote peer d2e1fdaeeab841af80053f63388320b1: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
23:51:57.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:57.897385 31729 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
23:51:58.135 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:51:58.135 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:51:58.135 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1860), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [19ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [38ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [40ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [41ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [41ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [59ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [60ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [61ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [61ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [159ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [160ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [162ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [162ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [536ms] refreshing cache from master, [536ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [539ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [541ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [542ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [542ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [1855ms] refreshing cache from master, [1856ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1859ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@254827604(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5ea30e23 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149) -> (continuation of Deferred@761183401 after retry RPC@28859825) -> (continuation of Deferred@1725799358 after retry RPC@625673985) -> (continuation of Deferred@82231522 after retry RPC@2045291632), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e0c7fbb -> wakeup thread Time-limited test -> (continuation of Deferred@1824574301 after retry RPC@298619097) -> (continuation of Deferred@256234852 after retry RPC@2063259149) -> (continuation of Deferred@761183401 after retry RPC@28859825) -> (continuation of Deferred@1725799358 after retry RPC@625673985) -> (continuation of Deferred@82231522 after retry RPC@2045291632))) after lookup
23:51:58.136 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:51:58.137 [DEBUG - kudu-nio-1] (Connection.java:757) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] Successfully connected to peer
23:51:58.159 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xba02bd49, L:/127.0.0.1:55454 - R:/127.29.219.193:45855] using TOKEN/null
23:51:58.205 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.195:43891
23:51:58.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:58.206657 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:43891
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33295
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:58.538551 31759 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:51:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:58.539001 31759 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:58.539448 31759 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:51:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:58.571067 31759 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:51:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:58.572134 31759 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.195
23:51:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:58.607486 31759 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:43891
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33295
23:51:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:58.609447 31759 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:58.611244 31759 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:58.624574 31766 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.282850 31537 debug-util.cc:398] Leaking SignalData structure 0x7b08000bfa40 after lost signal to thread 31407
23:51:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.284353 31537 debug-util.cc:398] Leaking SignalData structure 0x7b08000c6020 after lost signal to thread 31540
23:51:59.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:58.625422 31767 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.941432 31768 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1316 milliseconds
23:51:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.941671 31759 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:51:59.941699 31769 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.945382 31759 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:59.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.947893 31759 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:59.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.949247 31759 hybrid_clock.cc:648] HybridClock initialized: now 1783468319949196 us; error 59 us; skew 500 ppm
23:51:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.952102 31759 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.965803 31759 webserver.cc:534] Webserver started at http://127.29.219.195:33295/ using document root <none> and password file <none>
23:51:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.966691 31759 fs_manager.cc:362] Metadata directory not provided
23:51:59.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.966908 31759 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.975443 31759 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.004s
23:51:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.981768 31781 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:51:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.983034 31759 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.003s
23:51:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.983422 31759 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:51:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:42 on dist-test-slave-hdrm"
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:51:59.985431 31759 fs_report.cc:389] FS layout report
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/data
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.051460 31759 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.053475 31759 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:00.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.053992 31759 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.057299 31759 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.064077 31791 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.072279 31759 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.072515 31759 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.001s
23:52:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.072860 31759 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.077456 31759 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.077697 31759 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
23:52:00.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.078086 31791 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap starting.
23:52:00.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.152920 31791 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Log is configured to *not* fsync() on all Append() calls
23:52:00.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.217883 31791 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:00.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.219458 31791 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap complete.
23:52:00.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.221592 31791 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent bootstrapping tablet: real 0.144s	user 0.080s	sys 0.058s
23:52:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.250850 31791 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.254814 31791 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Initialized, Role: FOLLOWER
23:52:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.256273 31791 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.262542 31791 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent starting tablet: real 0.040s	user 0.035s	sys 0.003s
23:52:00.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.317062 31759 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.195:43891
23:52:00.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.317404 31900 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.195:43891 every 8 connection(s)
23:52:00.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.319784 31759 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:52:00.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.327672 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 31759
23:52:00.329 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:00.372 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:00.406 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.193:45855
23:52:00.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.407532 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 31406
23:52:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.410934 31902 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:52:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.411445 31902 heartbeater.cc:461] Registering TS with master...
23:52:00.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.412789 31902 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:52:00.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.417368 30696 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.424085 31903 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:52:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.424499 31903 heartbeater.cc:461] Registering TS with master...
23:52:00.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.425544 31903 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:52:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.429679 30620 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.435509 31901 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:52:00.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.435914 31901 heartbeater.cc:461] Registering TS with master...
23:52:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.436954 31901 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:52:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.435303 30620 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.195:43953
23:52:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:00.440908 30775 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:00.459 [DEBUG - kudu-nio-1] (Connection.java:239) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] handling channelInactive
23:52:00.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state READY due to: connection closed
23:52:00.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:00.460 [DEBUG - Time-limited test] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:00.461 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.461 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.461 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.474 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:00.475 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:00.475 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [14ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@235973424(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5c090032 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c8bddad -> wakeup thread Time-limited test)) after lookup
23:52:00.476 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:00.477 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.477 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.477 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.495 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:00.495 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:00.496 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [14ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [15ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [17ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [17ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@235973424(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5c090032 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c8bddad -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836))) after lookup
23:52:00.496 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:00.498 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.498 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.498 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.535 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:00.535 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:00.535 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [14ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [15ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [17ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [17ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [36ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [38ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [38ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@235973424(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5c090032 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c8bddad -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579))) after lookup
23:52:00.536 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:00.537 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.538 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.538 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.594 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:00.595 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:00.595 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=135), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [14ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [15ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [17ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [17ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [36ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [38ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [38ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [76ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [78ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [78ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [130ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [134ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@235973424(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5c090032 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c8bddad -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486))) after lookup
23:52:00.596 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:00.597 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.597 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.597 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.775 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:00.775 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:00.775 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=315), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [14ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [15ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [17ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [17ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [36ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [38ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [38ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [76ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [78ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [78ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [130ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [134ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [136ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [137ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [137ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [311ms] refreshing cache from master, [311ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [314ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@235973424(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5c090032 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486) -> (continuation of Deferred@1694570327 after retry RPC@391023731), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c8bddad -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486) -> (continuation of Deferred@1694570327 after retry RPC@391023731))) after lookup
23:52:00.776 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:00.777 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.777 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:00.778 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:01.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.444010 31903 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:52:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.488549 31913 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:52:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.489153 31913 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.492270 31913 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.499647 31785 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.500788 31917 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:52:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.501305 31917 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.511029 31917 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.513216 31673 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d" is_pre_election: true
23:52:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.514225 31673 raft_consensus.cc:2393] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 0edbfeb3c4ab49e1b0da501a288438f4 in current term 4: Already voted for candidate d2e1fdaeeab841af80053f63388320b1 in this term.
23:52:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.515100 31785 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.516454 31788 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4; no voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1
23:52:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.517480 31913 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 3 FOLLOWER]: Advancing to term 4
23:52:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.520159 31602 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.526307 31602 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:01.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.528000 31913 raft_consensus.cc:2749] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 4 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
23:52:01.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.527765 31854 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "451a779f984c42ac861a7b36231ac58d" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" is_pre_election: true
23:52:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.528654 31854 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 451a779f984c42ac861a7b36231ac58d in term 4.
23:52:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.529851 31602 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:52:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.530440 31917 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Leader pre-election won for term 5
23:52:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.530717 31917 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Starting leader election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:52:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.531036 31917 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 4 FOLLOWER]: Advancing to term 5
23:52:01.535 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:01.535 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:01.536 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1076), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [14ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [15ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [17ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [17ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [36ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [38ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [38ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [76ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [78ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [78ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [130ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [134ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [136ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [137ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [137ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [311ms] refreshing cache from master, [311ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [314ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [316ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [318ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [318ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1070ms] refreshing cache from master, [1071ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1075ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@235973424(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5c090032 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486) -> (continuation of Deferred@1694570327 after retry RPC@391023731) -> (continuation of Deferred@1329152592 after retry RPC@746021584), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c8bddad -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486) -> (continuation of Deferred@1694570327 after retry RPC@391023731) -> (continuation of Deferred@1329152592 after retry RPC@746021584))) after lookup
23:52:01.537 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:01.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.537498 31917 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:01.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:01.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:01.538 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:01.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.539351 31917 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 5 election: Requested vote from peers 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:01.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.540326 31854 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "451a779f984c42ac861a7b36231ac58d" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:52:01.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.540798 31854 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 4 FOLLOWER]: Advancing to term 5
23:52:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:01.543256 31602 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.548789 31854 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 451a779f984c42ac861a7b36231ac58d in term 5.
23:52:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.549891 31602 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:52:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.550534 31917 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 5 FOLLOWER]: Leader election won for term 5
23:52:01.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.552534 31917 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 5 LEADER]: Becoming Leader. State: Replica: 451a779f984c42ac861a7b36231ac58d, State: Running, Role: LEADER
23:52:01.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.554293 31917 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:01.564553 30620 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d reported cstate change: term changed from 4 to 5, leader changed from d2e1fdaeeab841af80053f63388320b1 (127.29.219.193) to 451a779f984c42ac861a7b36231ac58d (127.29.219.194). New cstate: current_term: 5 leader_uuid: "451a779f984c42ac861a7b36231ac58d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: UNKNOWN } } }
23:52:02.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.020632 31854 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 5 FOLLOWER]: Refusing update from remote peer 451a779f984c42ac861a7b36231ac58d: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
23:52:02.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:02.022045 31917 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
23:52:02.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:02.054955 31602 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:04.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.187983 30729 debug-util.cc:398] Leaking SignalData structure 0x7b080003c240 after lost signal to thread 30662
23:52:04.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.188616 30729 debug-util.cc:398] Leaking SignalData structure 0x7b0800059720 after lost signal to thread 30732
23:52:04.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:04.718843 31602 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:52:05.336 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:05.337 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:05.337 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=4877), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [14ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [15ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [17ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [17ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [31ms] refreshing cache from master, [31ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [35ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [36ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [38ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [38ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [70ms] refreshing cache from master, [71ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [74ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [76ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [78ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [78ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [130ms] refreshing cache from master, [131ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [134ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [136ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [137ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [137ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [311ms] refreshing cache from master, [311ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [314ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [316ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [318ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [318ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1070ms] refreshing cache from master, [1071ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1075ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [1076ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1078ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [1078ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [4871ms] refreshing cache from master, [4872ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [4876ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@235973424(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@5c090032 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486) -> (continuation of Deferred@1694570327 after retry RPC@391023731) -> (continuation of Deferred@1329152592 after retry RPC@746021584) -> (continuation of Deferred@1602600586 after retry RPC@547537257), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6c8bddad -> wakeup thread Time-limited test -> (continuation of Deferred@1114228737 after retry RPC@1883050836) -> (continuation of Deferred@1678730533 after retry RPC@2083552579) -> (continuation of Deferred@1263365361 after retry RPC@887671486) -> (continuation of Deferred@1694570327 after retry RPC@391023731) -> (continuation of Deferred@1329152592 after retry RPC@746021584) -> (continuation of Deferred@1602600586 after retry RPC@547537257))) after lookup
23:52:05.338 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:05.339 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] Successfully connected to peer
23:52:05.361 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6d8e3656, L:/127.0.0.1:57364 - R:/127.29.219.194:41145] using TOKEN/null
23:52:05.418 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.193:45855
23:52:05.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:05.420504 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:05.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.778208 31965 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.778693 31965 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.779256 31965 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.812098 31965 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.812904 31965 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.193
23:52:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:05.850042 31965 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:52:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:52:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:05.851785 31965 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:05.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:05.853677 31965 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:05.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.867933 31971 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:06.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:06.920415 31602 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111) [suppressed 10 similar messages]
23:52:07.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:05.869180 31972 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:07.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:07.132738 31973 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:52:07.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:07.134509 31974 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:07.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:07.137094 31965 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.269s	user 0.518s	sys 0.749s
23:52:07.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:07.137531 31965 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.270s	user 0.518s	sys 0.749s
23:52:07.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.137836 31965 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:52:07.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.139396 31965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:07.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.142215 31965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:07.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.143684 31965 hybrid_clock.cc:648] HybridClock initialized: now 1783468327143632 us; error 54 us; skew 500 ppm
23:52:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.144901 31965 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:52:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.153664 31965 webserver.cc:534] Webserver started at http://127.29.219.193:43571/ using document root <none> and password file <none>
23:52:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.154961 31965 fs_manager.cc:362] Metadata directory not provided
23:52:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.155323 31965 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:07.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.166893 31965 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.005s	sys 0.001s
23:52:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.172806 31985 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.174188 31965 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.005s
23:52:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.174649 31965 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2e1fdaeeab841af80053f63388320b1"
23:52:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:38 on dist-test-slave-hdrm"
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.177749 31965 fs_report.cc:389] FS layout report
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/data
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.255494 31965 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.257102 31965 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.257625 31965 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.260308 31965 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.266908 31992 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.274860 31965 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.275060 31965 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
23:52:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.275282 31965 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.280005 31965 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.280261 31965 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.005s	sys 0.000s
23:52:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.280593 31992 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap starting.
23:52:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.358712 31992 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Log is configured to *not* fsync() on all Append() calls
23:52:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.432519 31992 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:07.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.433537 31992 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap complete.
23:52:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.435411 31992 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent bootstrapping tablet: real 0.155s	user 0.119s	sys 0.030s
23:52:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:07.459455 31602 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
23:52:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.460304 31965 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.193:45855
23:52:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.460438 32102 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.193:45855 every 8 connection(s)
23:52:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.463076 31965 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.461769 31992 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.464120 31992 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Initialized, Role: FOLLOWER
23:52:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.465058 31992 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.469115 31992 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent starting tablet: real 0.033s	user 0.032s	sys 0.000s
23:52:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.474620 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 31965
23:52:07.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.512722 32103 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:52:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.513188 32103 heartbeater.cc:461] Registering TS with master...
23:52:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.514457 32103 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:52:07.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.518760 30774 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.540551 32105 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:52:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.540951 32105 heartbeater.cc:461] Registering TS with master...
23:52:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.540904 32106 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:52:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.541308 32106 heartbeater.cc:461] Registering TS with master...
23:52:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.541949 32105 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:52:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.542187 32106 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:52:07.544 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:07.545 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.194:41145
23:52:07.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.546612 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 31583
23:52:07.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.546880 30619 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:07.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.549749 30696 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:07.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:07.550036 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.193:52371
23:52:07.576 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] handling channelInactive
23:52:07.576 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state READY due to: connection closed
23:52:07.577 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:07.577 [DEBUG - Time-limited test] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:07.578 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.579 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.579 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.595 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:07.595 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:07.595 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@991828299(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32eeac24 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34ebfa1f -> wakeup thread Time-limited test)) after lookup
23:52:07.596 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:07.597 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.597 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.597 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.614 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:07.615 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:07.615 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [19ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [20ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [21ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@991828299(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32eeac24 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34ebfa1f -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299))) after lookup
23:52:07.616 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:07.617 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.617 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.617 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.655 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:07.656 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:07.656 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [19ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [20ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [21ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [40ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [40ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [78ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@991828299(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32eeac24 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34ebfa1f -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848))) after lookup
23:52:07.657 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:07.658 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.658 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.658 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.755 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:07.755 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:07.756 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=178), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [19ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [20ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [21ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [40ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [40ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [78ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [79ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [81ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [81ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [174ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [178ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@991828299(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32eeac24 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848) -> (continuation of Deferred@568231437 after retry RPC@1783486178), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34ebfa1f -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848) -> (continuation of Deferred@568231437 after retry RPC@1783486178))) after lookup
23:52:07.756 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:07.757 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.757 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:07.758 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:08.075 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:08.075 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:08.075 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=498), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [19ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [20ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [21ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [40ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [40ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [78ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [79ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [81ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [81ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [174ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [178ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [179ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [181ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [181ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [494ms] refreshing cache from master, [495ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [498ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@991828299(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32eeac24 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848) -> (continuation of Deferred@568231437 after retry RPC@1783486178) -> (continuation of Deferred@889491344 after retry RPC@830121236), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34ebfa1f -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848) -> (continuation of Deferred@568231437 after retry RPC@1783486178) -> (continuation of Deferred@889491344 after retry RPC@830121236))) after lookup
23:52:08.076 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:08.077 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:08.077 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:08.077 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.553701 32106 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:52:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.608840 32117 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:52:08.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.609453 32117 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.611992 32117 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:08.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.615959 31989 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.620059 31989 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.624317 31854 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" is_pre_election: true
23:52:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.626022 31986 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d2e1fdaeeab841af80053f63388320b1; no voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d
23:52:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.626770 32117 raft_consensus.cc:2749] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
23:52:08.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.916467 32122 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 451a779f984c42ac861a7b36231ac58d)
23:52:08.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.917001 32122 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.919060 32122 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.922482 31788 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:08.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.926476 31788 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.931768 32055 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "d2e1fdaeeab841af80053f63388320b1" is_pre_election: true
23:52:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.932513 32055 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 4.
23:52:08.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.933750 31785 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1; no voters: 451a779f984c42ac861a7b36231ac58d
23:52:08.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.934419 32122 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 5 FOLLOWER]: Leader pre-election won for term 6
23:52:08.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.934696 32122 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 451a779f984c42ac861a7b36231ac58d)
23:52:08.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.935001 32122 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 5 FOLLOWER]: Advancing to term 6
23:52:08.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.939569 32122 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:08.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.941244 32122 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 6 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.942196 32055 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "d2e1fdaeeab841af80053f63388320b1"
23:52:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.942642 32055 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 4 FOLLOWER]: Advancing to term 6
23:52:08.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:08.944476 31788 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.949556 32055 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 6.
23:52:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.950368 31785 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1; no voters: 451a779f984c42ac861a7b36231ac58d
23:52:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.950971 32122 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 6 FOLLOWER]: Leader election won for term 6
23:52:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.952172 32122 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 6 LEADER]: Becoming Leader. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Running, Role: LEADER
23:52:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.953671 32122 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:08.962450 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 reported cstate change: term changed from 5 to 6, leader changed from 451a779f984c42ac861a7b36231ac58d (127.29.219.194) to 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195). New cstate: current_term: 6 leader_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: UNKNOWN } } }
23:52:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:09.427340 31788 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Couldn't send request to peer 451a779f984c42ac861a7b36231ac58d. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:09.465762 32055 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 6 FOLLOWER]: Refusing update from remote peer 0edbfeb3c4ab49e1b0da501a288438f4: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
23:52:09.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:09.467581 32122 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
23:52:10.035 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:10.036 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:10.036 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2459), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [2ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [19ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [20ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [21ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [40ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [40ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [78ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [79ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [81ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [81ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [174ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [178ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [179ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [181ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [181ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [494ms] refreshing cache from master, [495ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [498ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [499ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [500ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [500ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [2454ms] refreshing cache from master, [2454ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [2458ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@991828299(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32eeac24 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848) -> (continuation of Deferred@568231437 after retry RPC@1783486178) -> (continuation of Deferred@889491344 after retry RPC@830121236) -> (continuation of Deferred@1232174041 after retry RPC@768030182), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@34ebfa1f -> wakeup thread Time-limited test -> (continuation of Deferred@194020588 after retry RPC@1614855299) -> (continuation of Deferred@1484773360 after retry RPC@1027525848) -> (continuation of Deferred@568231437 after retry RPC@1783486178) -> (continuation of Deferred@889491344 after retry RPC@830121236) -> (continuation of Deferred@1232174041 after retry RPC@768030182))) after lookup
23:52:10.037 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:10.038 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] Successfully connected to peer
23:52:10.057 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4e1fd4eb, L:/127.0.0.1:44938 - R:/127.29.219.195:43891] using TOKEN/null
23:52:10.115 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.194:41145
23:52:10.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.118019 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:41145
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36759
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.445534 32156 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.446059 32156 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.446511 32156 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.479929 32156 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.480901 32156 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.194
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.517541 32156 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:41145
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36759
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:52:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:52:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.519304 32156 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:10.521124 32156 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:10.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.537444 32164 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:11.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:11.893396 31788 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Couldn't send request to peer 451a779f984c42ac861a7b36231ac58d. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:52:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:11.953401 32163 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 32156
23:52:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.300433 32163 kernel_stack_watchdog.cc:198] Thread 32156 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 410ms:
23:52:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:52:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:52:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:52:12.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:52:12.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:10.538296 32165 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:12.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.301445 32156 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.763s	user 0.474s	sys 1.234s
23:52:12.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.301966 32156 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.764s	user 0.475s	sys 1.234s
23:52:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.303699 32168 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.306116 32166 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1768 milliseconds
23:52:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.306138 32156 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:52:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.307458 32156 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:12.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.309531 32156 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.310914 32156 hybrid_clock.cc:648] HybridClock initialized: now 1783468332310877 us; error 35 us; skew 500 ppm
23:52:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.311764 32156 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:52:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.318425 32156 webserver.cc:534] Webserver started at http://127.29.219.194:36759/ using document root <none> and password file <none>
23:52:12.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.319743 32156 fs_manager.cc:362] Metadata directory not provided
23:52:12.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.320037 32156 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.328665 32156 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.000s
23:52:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.333087 32178 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.334321 32156 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
23:52:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.334601 32156 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451a779f984c42ac861a7b36231ac58d"
23:52:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:40 on dist-test-slave-hdrm"
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.336544 32156 fs_report.cc:389] FS layout report
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/data
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:12.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.394938 32156 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:12.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.396521 32156 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.396924 32156 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.399449 32156 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.405427 32185 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.413524 32156 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.413735 32156 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
23:52:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.413945 32156 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.418491 32156 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.418679 32156 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
23:52:12.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.419247 32185 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap starting.
23:52:12.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.485127 32185 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Log is configured to *not* fsync() on all Append() calls
23:52:12.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.556803 30806 debug-util.cc:398] Leaking SignalData structure 0x7b0800097160 after lost signal to thread 30740
23:52:12.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:12.557524 30806 debug-util.cc:398] Leaking SignalData structure 0x7b0800098ec0 after lost signal to thread 30809
23:52:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.580610 32185 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.581873 32185 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap complete.
23:52:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.584060 32185 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent bootstrapping tablet: real 0.165s	user 0.100s	sys 0.054s
23:52:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.612594 32185 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.616149 32185 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 451a779f984c42ac861a7b36231ac58d, State: Initialized, Role: FOLLOWER
23:52:12.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.617455 32185 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:12.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.624668 32185 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent starting tablet: real 0.040s	user 0.027s	sys 0.013s
23:52:12.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.635612 32156 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.194:41145
23:52:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.636736 32294 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.194:41145 every 8 connection(s)
23:52:12.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.640067 32156 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:52:12.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.647123 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 32156
23:52:12.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:12.704 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:12.705 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.195:43891
23:52:12.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.705698 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 31759
23:52:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.731905 32300 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:52:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.732400 32300 heartbeater.cc:461] Registering TS with master...
23:52:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.732671 32295 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:52:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.733047 32295 heartbeater.cc:461] Registering TS with master...
23:52:12.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.733870 32300 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:52:12.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.734316 32295 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:52:12.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.738772 30774 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:52:12.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.739327 30619 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:52:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.741178 32296 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:52:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.741551 32296 heartbeater.cc:461] Registering TS with master...
23:52:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.742486 32296 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:52:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.742606 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.194:41667
23:52:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:12.748772 30696 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:52:12.763 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] handling channelInactive
23:52:12.764 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state READY due to: connection closed
23:52:12.763 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:12.765 [DEBUG - Time-limited test] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:12.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.767 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.775 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:12.775 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:12.775 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=10), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [10ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1905978759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6fb813de -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5bcf1b88 -> wakeup thread Time-limited test)) after lookup
23:52:12.776 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:12.777 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.777 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.777 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.794 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:12.795 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:12.795 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [10ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [11ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [12ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [12ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1905978759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6fb813de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5bcf1b88 -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574))) after lookup
23:52:12.796 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:12.798 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.798 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.798 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.835 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:12.835 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:12.836 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=70), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [10ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [11ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [12ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [12ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [30ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [33ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [33ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1905978759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6fb813de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5bcf1b88 -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933))) after lookup
23:52:12.836 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:12.838 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.838 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.838 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.876 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:12.876 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:12.876 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=111), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [10ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [11ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [12ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [12ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [30ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [33ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [33ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [71ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [73ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [73ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [110ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1905978759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6fb813de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933) -> (continuation of Deferred@355770553 after retry RPC@2073519097), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5bcf1b88 -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933) -> (continuation of Deferred@355770553 after retry RPC@2073519097))) after lookup
23:52:12.877 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:12.878 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.879 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:12.879 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:13.196 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:13.196 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:13.197 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=432), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [10ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [11ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [12ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [12ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [30ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [33ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [33ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [71ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [73ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [73ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [110ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [112ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [114ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [114ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [426ms] refreshing cache from master, [426ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [431ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1905978759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6fb813de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933) -> (continuation of Deferred@355770553 after retry RPC@2073519097) -> (continuation of Deferred@1764401455 after retry RPC@702713688), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5bcf1b88 -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933) -> (continuation of Deferred@355770553 after retry RPC@2073519097) -> (continuation of Deferred@1764401455 after retry RPC@702713688))) after lookup
23:52:13.198 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:13.199 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:13.199 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:13.199 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:13.747407 32300 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:52:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.167315 32307 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 0edbfeb3c4ab49e1b0da501a288438f4)
23:52:14.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.167698 32307 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:14.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.175474 32307 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:14.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.186074 31986 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:52:14.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.189285 31986 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:52:14.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.194226 32249 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d" is_pre_election: true
23:52:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.195016 32249 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 5.
23:52:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.196372 31989 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1; no voters: 0edbfeb3c4ab49e1b0da501a288438f4
23:52:14.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.197038 32307 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 6 FOLLOWER]: Leader pre-election won for term 7
23:52:14.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.197369 32307 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 0edbfeb3c4ab49e1b0da501a288438f4)
23:52:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.197726 32307 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 6 FOLLOWER]: Advancing to term 7
23:52:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.202173 32307 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.203872 32307 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 7 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.204614 32249 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d"
23:52:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.205204 32249 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 5 FOLLOWER]: Advancing to term 7
23:52:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.207535 31986 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:52:14.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.215330 32249 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 7.
23:52:14.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.216401 31989 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1; no voters: 0edbfeb3c4ab49e1b0da501a288438f4
23:52:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.217031 32307 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 7 FOLLOWER]: Leader election won for term 7
23:52:14.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.218216 32307 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 7 LEADER]: Becoming Leader. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Running, Role: LEADER
23:52:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.219450 32307 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.229475 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 reported cstate change: term changed from 6 to 7, leader changed from 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195) to d2e1fdaeeab841af80053f63388320b1 (127.29.219.193). New cstate: current_term: 7 leader_uuid: "d2e1fdaeeab841af80053f63388320b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: HEALTHY } } }
23:52:14.395 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:14.395 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:14.396 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1630), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [10ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [11ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [12ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [12ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [29ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [30ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [33ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [33ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [69ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [71ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [73ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [73ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [106ms] refreshing cache from master, [106ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [110ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [112ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [114ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [114ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [426ms] refreshing cache from master, [426ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [431ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [433ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [434ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [434ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [1626ms] refreshing cache from master, [1626ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1630ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1905978759(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6fb813de -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933) -> (continuation of Deferred@355770553 after retry RPC@2073519097) -> (continuation of Deferred@1764401455 after retry RPC@702713688) -> (continuation of Deferred@1247296328 after retry RPC@2013127956), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5bcf1b88 -> wakeup thread Time-limited test -> (continuation of Deferred@288785443 after retry RPC@847695574) -> (continuation of Deferred@406549148 after retry RPC@642596933) -> (continuation of Deferred@355770553 after retry RPC@2073519097) -> (continuation of Deferred@1764401455 after retry RPC@702713688) -> (continuation of Deferred@1247296328 after retry RPC@2013127956))) after lookup
23:52:14.396 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:14.397 [DEBUG - kudu-nio-7] (Connection.java:757) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] Successfully connected to peer
23:52:14.419 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0xd213436d, L:/127.0.0.1:45056 - R:/127.29.219.193:45855] using TOKEN/null
23:52:14.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.428413 32249 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 7 FOLLOWER]: Refusing update from remote peer d2e1fdaeeab841af80053f63388320b1: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
23:52:14.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.429817 31986 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 -> Peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Couldn't send request to peer 0edbfeb3c4ab49e1b0da501a288438f4. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:14.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.430075 32312 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.001s
23:52:14.506 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.195:43891
23:52:14.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.508396 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:43891
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33295
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:52:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:14.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:14.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:14.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:14.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.820717 32335 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:14.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.821270 32335 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:14.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.821893 32335 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.854502 32335 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.855489 32335 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.195
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.892614 32335 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:43891
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:52:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33295
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:52:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.894402 32335 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:14.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:14.896435 32335 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:14.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.909821 32341 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:16.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:16.313180 32340 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 32335
23:52:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:16.368602 32335 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.459s	user 0.512s	sys 0.919s
23:52:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:16.369025 32335 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.459s	user 0.512s	sys 0.919s
23:52:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:14.910759 32342 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:16.370496 32343 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1459 milliseconds
23:52:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:16.371351 32344 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.371284 32335 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:52:16.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.375514 32335 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:16.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.377516 32335 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:16.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.378971 32335 hybrid_clock.cc:648] HybridClock initialized: now 1783468336378937 us; error 42 us; skew 500 ppm
23:52:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.379920 32335 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:52:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.386101 32335 webserver.cc:534] Webserver started at http://127.29.219.195:33295/ using document root <none> and password file <none>
23:52:16.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.387017 32335 fs_manager.cc:362] Metadata directory not provided
23:52:16.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.387259 32335 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:16.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.395025 32335 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
23:52:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.399708 32358 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.400820 32335 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.401183 32335 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:42 on dist-test-slave-hdrm"
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.403077 32335 fs_report.cc:389] FS layout report
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/data
23:52:16.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.460868 32335 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.462854 32335 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.463408 32335 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:16.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.466830 32335 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:16.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.474349 32365 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.482466 32335 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.483048 32335 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.001s
23:52:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.483383 32335 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.489521 32335 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.489758 32335 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.005s	sys 0.000s
23:52:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.490115 32365 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap starting.
23:52:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.570886 32365 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Log is configured to *not* fsync() on all Append() calls
23:52:16.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.653105 32365 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:16.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.654062 32365 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap complete.
23:52:16.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.656091 32365 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent bootstrapping tablet: real 0.166s	user 0.106s	sys 0.054s
23:52:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.683432 32335 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.195:43891
23:52:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.683971 32473 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.195:43891 every 8 connection(s)
23:52:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.681835 32365 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.685184 32365 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Initialized, Role: FOLLOWER
23:52:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.686177 32365 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.686545 32335 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:52:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.689770 32365 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent starting tablet: real 0.033s	user 0.032s	sys 0.000s
23:52:16.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.696785 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 32335
23:52:16.699 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:16.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.761675 32476 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:52:16.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.762010 32475 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:52:16.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.762230 32476 heartbeater.cc:461] Registering TS with master...
23:52:16.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.762436 32475 heartbeater.cc:461] Registering TS with master...
23:52:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.763667 32475 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:52:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.763780 32476 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:52:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.768402 30696 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.768980 32428 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 6 FOLLOWER]: Advancing to term 7
23:52:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.769098 30775 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.773608 32477 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:52:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.773955 32477 heartbeater.cc:461] Registering TS with master...
23:52:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.774931 32477 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:52:16.777 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:16.778 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.193:45855
23:52:16.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:16.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.778810 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 31965
23:52:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.809571 30619 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.812188 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.195:60575
23:52:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:16.813568 32360 connection.cc:573] server connection from 127.29.219.193:60429 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:52:16.814 [DEBUG - kudu-nio-7] (Connection.java:239) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] handling channelInactive
23:52:16.814 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state READY due to: connection closed
23:52:16.816 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:16.817 [DEBUG - Time-limited test] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:16.818029 32477 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:52:16.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.819 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.835 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:16.835 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:16.835 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1717227480(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@7e175342 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60499070 -> wakeup thread Time-limited test)) after lookup
23:52:16.835 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:16.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.836 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.854 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:16.855 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:16.855 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [19ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [19ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1717227480(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@7e175342 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60499070 -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396))) after lookup
23:52:16.855 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:16.856 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.856 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.857 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.874 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:16.874 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:16.874 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [19ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [19ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [40ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [40ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [56ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1717227480(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@7e175342 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60499070 -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576))) after lookup
23:52:16.874 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:16.875 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.875 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.876 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.894 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:16.894 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:16.895 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=78), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [19ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [19ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [40ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [40ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [56ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [57ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [59ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [59ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [77ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1717227480(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@7e175342 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576) -> (continuation of Deferred@900953954 after retry RPC@64291316), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60499070 -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576) -> (continuation of Deferred@900953954 after retry RPC@64291316))) after lookup
23:52:16.895 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:16.896 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.896 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:16.896 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:17.095 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:17.095 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:17.095 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=278), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [19ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [19ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [40ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [40ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [56ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [57ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [59ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [59ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [77ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [78ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [79ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [79ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [274ms] refreshing cache from master, [274ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [277ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1717227480(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@7e175342 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576) -> (continuation of Deferred@900953954 after retry RPC@64291316) -> (continuation of Deferred@160271148 after retry RPC@578609480), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60499070 -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576) -> (continuation of Deferred@900953954 after retry RPC@64291316) -> (continuation of Deferred@160271148 after retry RPC@578609480))) after lookup
23:52:17.096 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:17.097 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:17.097 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:17.097 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:18.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.026990 32492 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:52:18.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.027519 32492 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:18.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.029985 32492 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:18.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.033687 32179 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.036859 32179 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.038883 32428 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "451a779f984c42ac861a7b36231ac58d" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" is_pre_election: true
23:52:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.040400 32179 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d; no voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1
23:52:18.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.041036 32492 raft_consensus.cc:2749] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
23:52:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.283666 32497 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:52:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.284214 32497 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.286955 32497 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:18.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.294271 32359 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:18.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.298663 32359 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.301457 32249 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d" is_pre_election: true
23:52:18.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.301975 32249 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 7.
23:52:18.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.303076 32362 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:52:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.303779 32497 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 7 FOLLOWER]: Leader pre-election won for term 8
23:52:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.304121 32497 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 7 FOLLOWER]: Starting leader election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:52:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.304343 32497 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 7 FOLLOWER]: Advancing to term 8
23:52:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.308616 32497 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.310109 32497 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 8 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.310675 32249 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d"
23:52:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.311187 32249 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 7 FOLLOWER]: Advancing to term 8
23:52:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.313835 32359 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:18.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.317445 32249 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 8.
23:52:18.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.318363 32362 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:52:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.318972 32497 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 8 FOLLOWER]: Leader election won for term 8
23:52:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.320550 32497 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 8 LEADER]: Becoming Leader. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Running, Role: LEADER
23:52:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.321296 32497 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.329918 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 reported cstate change: term changed from 7 to 8, leader changed from d2e1fdaeeab841af80053f63388320b1 (127.29.219.193) to 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195). New cstate: current_term: 8 leader_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: UNKNOWN } } }
23:52:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.638943 30729 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1ca0 after lost signal to thread 30662
23:52:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.639609 30729 debug-util.cc:398] Leaking SignalData structure 0x7b0800093780 after lost signal to thread 30732
23:52:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:18.794754 32359 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:18.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.867646 32249 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 8 FOLLOWER]: Refusing update from remote peer 0edbfeb3c4ab49e1b0da501a288438f4: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
23:52:18.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:18.868986 32497 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 28, Time since last communication: 0.000s
23:52:18.935 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:18.936 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:18.936 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2119), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [19ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [19ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [40ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [40ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [56ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [57ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [59ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [59ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [74ms] refreshing cache from master, [74ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [77ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [78ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [79ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [79ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [274ms] refreshing cache from master, [274ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [277ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [279ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [280ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [280ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2114ms] refreshing cache from master, [2115ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [2118ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1717227480(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@7e175342 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576) -> (continuation of Deferred@900953954 after retry RPC@64291316) -> (continuation of Deferred@160271148 after retry RPC@578609480) -> (continuation of Deferred@2109684548 after retry RPC@948761370), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@60499070 -> wakeup thread Time-limited test -> (continuation of Deferred@2109370454 after retry RPC@1564535396) -> (continuation of Deferred@1763271837 after retry RPC@869314576) -> (continuation of Deferred@900953954 after retry RPC@64291316) -> (continuation of Deferred@160271148 after retry RPC@578609480) -> (continuation of Deferred@2109684548 after retry RPC@948761370))) after lookup
23:52:18.936 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:18.937 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] Successfully connected to peer
23:52:18.962 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x71501dea, L:/127.0.0.1:47620 - R:/127.29.219.195:43891] using TOKEN/null
23:52:19.006 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.193:45855
23:52:19.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:19.008281 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:52:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.316218 32524 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.316761 32524 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.317543 32524 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.352387 32524 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.353356 32524 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.193
23:52:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:19.395398 32524 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:52:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:52:19.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:19.396817 32524 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:19.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:19.398361 32524 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.410671 32532 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:21.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:20.815409 32531 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 32524
23:52:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:21.202395 32531 kernel_stack_watchdog.cc:198] Thread 32524 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 400ms:
23:52:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:52:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:52:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:52:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:52:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:19.412731 32533 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:21.203927 32524 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.792s	user 0.618s	sys 1.129s
23:52:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:21.204252 32524 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.792s	user 0.618s	sys 1.129s
23:52:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:21.205168 32535 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:21.207894 32534 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1791 milliseconds
23:52:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.207914 32524 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:52:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.209096 32524 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.211071 32524 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.212392 32524 hybrid_clock.cc:648] HybridClock initialized: now 1783468341212355 us; error 76 us; skew 500 ppm
23:52:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.213150 32524 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:52:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.219887 32524 webserver.cc:534] Webserver started at http://127.29.219.193:43571/ using document root <none> and password file <none>
23:52:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.220700 32524 fs_manager.cc:362] Metadata directory not provided
23:52:21.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.220891 32524 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.231213 32524 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.000s
23:52:21.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.237960 32550 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.239374 32524 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.004s	sys 0.001s
23:52:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.239789 32524 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2e1fdaeeab841af80053f63388320b1"
23:52:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:38 on dist-test-slave-hdrm"
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.242650 32524 fs_report.cc:389] FS layout report
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/data
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:21.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.322496 32524 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.324012 32524 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.324525 32524 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.327832 32524 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.335018 32557 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.343906 32524 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.344115 32524 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.002s	sys 0.000s
23:52:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.344327 32524 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.348757 32524 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.349009 32524 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.004s	sys 0.000s
23:52:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.349356 32557 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap starting.
23:52:21.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.438802 32557 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Log is configured to *not* fsync() on all Append() calls
23:52:21.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:21.452309 32359 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:52:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.564420 32557 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:21.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.565562 32557 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap complete.
23:52:21.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.567478 32557 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent bootstrapping tablet: real 0.219s	user 0.171s	sys 0.040s
23:52:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.594961 32557 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.600122 32557 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Initialized, Role: FOLLOWER
23:52:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.601665 32557 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.607580 32557 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Time spent starting tablet: real 0.040s	user 0.025s	sys 0.015s
23:52:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.614683 32524 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.193:45855
23:52:21.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.614998 32666 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.193:45855 every 8 connection(s)
23:52:21.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.617367 32524 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:21.621 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:21.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.620709 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 32524
23:52:21.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.725471 32669 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:52:21.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.726164 32669 heartbeater.cc:461] Registering TS with master...
23:52:21.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.729082 32669 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:52:21.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.738057 30620 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.744354 30620 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.193:52509
23:52:21.749 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:21.750 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.195:43891
23:52:21.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:21.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.750780 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 32335
23:52:21.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.759616 32667 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:52:21.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.760087 32667 heartbeater.cc:461] Registering TS with master...
23:52:21.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.760967 32667 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:52:21.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.764426 30774 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:21.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.766301 32668 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:52:21.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.766696 32668 heartbeater.cc:461] Registering TS with master...
23:52:21.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.767294 32668 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:52:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:21.770229 30696 ts_manager.cc:194] Re-registered known tserver with Master: d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:21.791 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] handling channelInactive
23:52:21.792 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state READY due to: connection closed
23:52:21.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:21.795 [DEBUG - Time-limited test] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:21.796 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.797 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.797 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.815 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:21.815 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:21.815 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [20ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1835488657(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32ac7f04 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@530f3802 -> wakeup thread Time-limited test)) after lookup
23:52:21.816 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:21.817 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.817 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.817 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.835 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:21.835 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:21.835 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [20ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [39ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1835488657(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32ac7f04 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@530f3802 -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370))) after lookup
23:52:21.836 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:21.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.837 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.875 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:21.875 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:21.876 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [20ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [39ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [41ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [42ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [42ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1835488657(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32ac7f04 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@530f3802 -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975))) after lookup
23:52:21.876 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:21.877 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.877 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.877 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.955 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:21.955 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:21.955 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=160), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [20ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [39ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [41ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [42ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [42ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [81ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [82ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [82ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [159ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1835488657(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32ac7f04 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@530f3802 -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420))) after lookup
23:52:21.956 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:21.957 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.957 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:21.958 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:22.173929 30806 debug-util.cc:398] Leaking SignalData structure 0x7b0800099000 after lost signal to thread 30740
23:52:22.215 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:22.215 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:22.215 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=420), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [20ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [39ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [41ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [42ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [42ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [81ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [82ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [82ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [159ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [161ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [162ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [163ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [416ms] refreshing cache from master, [416ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [420ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1835488657(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32ac7f04 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420) -> (continuation of Deferred@271967694 after retry RPC@176644490), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@530f3802 -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420) -> (continuation of Deferred@271967694 after retry RPC@176644490))) after lookup
23:52:22.216 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:22.217 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:22.217 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:22.217 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:22.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.748974 32669 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:52:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.842212 32678 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 0edbfeb3c4ab49e1b0da501a288438f4)
23:52:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.842854 32678 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:22.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.844820 32678 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:22.853590 32179 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:52:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.858062 32621 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "451a779f984c42ac861a7b36231ac58d" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "d2e1fdaeeab841af80053f63388320b1" is_pre_election: true
23:52:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.858673 32621 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 451a779f984c42ac861a7b36231ac58d in term 7.
23:52:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.859694 32179 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1; no voters: 0edbfeb3c4ab49e1b0da501a288438f4
23:52:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.860322 32678 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 8 FOLLOWER]: Leader pre-election won for term 9
23:52:22.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.860626 32678 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 8 FOLLOWER]: Starting leader election (detected failure of leader 0edbfeb3c4ab49e1b0da501a288438f4)
23:52:22.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.860994 32678 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 8 FOLLOWER]: Advancing to term 9
23:52:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.865885 32678 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:22.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.867295 32678 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 9 election: Requested vote from peers 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:22.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.869041 32621 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "451a779f984c42ac861a7b36231ac58d" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "d2e1fdaeeab841af80053f63388320b1"
23:52:22.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.869550 32621 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 7 FOLLOWER]: Advancing to term 9
23:52:22.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:22.871035 32179 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111)
23:52:22.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.876295 32621 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 451a779f984c42ac861a7b36231ac58d in term 9.
23:52:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.877148 32179 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 451a779f984c42ac861a7b36231ac58d, d2e1fdaeeab841af80053f63388320b1; no voters: 0edbfeb3c4ab49e1b0da501a288438f4
23:52:22.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.877743 32678 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 9 FOLLOWER]: Leader election won for term 9
23:52:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.878878 32678 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 9 LEADER]: Becoming Leader. State: Replica: 451a779f984c42ac861a7b36231ac58d, State: Running, Role: LEADER
23:52:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.879850 32678 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:22.890050 30620 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d reported cstate change: term changed from 8 to 9, leader changed from 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195) to 451a779f984c42ac861a7b36231ac58d (127.29.219.194). New cstate: current_term: 9 leader_uuid: "451a779f984c42ac861a7b36231ac58d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: UNKNOWN } } }
23:52:22.895 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:22.895 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:22.896 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1100), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [20ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [39ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [41ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [42ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [42ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [81ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [82ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [82ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [159ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [161ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [162ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [163ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [416ms] refreshing cache from master, [416ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [420ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [421ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [422ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [422ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [1096ms] refreshing cache from master, [1096ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1100ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1835488657(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32ac7f04 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420) -> (continuation of Deferred@271967694 after retry RPC@176644490) -> (continuation of Deferred@647911117 after retry RPC@396687525), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@530f3802 -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420) -> (continuation of Deferred@271967694 after retry RPC@176644490) -> (continuation of Deferred@647911117 after retry RPC@396687525))) after lookup
23:52:22.897 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:22.900 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:22.900 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:22.900 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891
23:52:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:23.331548 32621 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 9 FOLLOWER]: Refusing update from remote peer 451a779f984c42ac861a7b36231ac58d: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
23:52:23.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:23.333092 32678 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
23:52:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:23.347648 32698 mvcc.cc:204] Tried to move back new op lower bound from 7305086316410363904 to 7305086313772240896. Current Snapshot: MvccSnapshot[applied={T|T < 7305086316410363904}]
23:52:23.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:23.365438 32179 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111) [suppressed 2 similar messages]
23:52:23.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:23.368732 32179 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d -> Peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Couldn't send request to peer 0edbfeb3c4ab49e1b0da501a288438f4. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:25.355 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:25.356 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:25.356 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=3561), Traces: [0ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [2ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [16ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [20ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [21ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [22ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [36ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [39ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [41ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [42ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [42ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [80ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [81ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [82ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [82ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [156ms] refreshing cache from master, [156ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [159ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [161ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [162ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [163ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [416ms] refreshing cache from master, [416ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [420ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [421ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [422ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [422ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [1096ms] refreshing cache from master, [1096ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1100ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [1101ms] sending RPC to server 0edbfeb3c4ab49e1b0da501a288438f4, [1105ms] received response from server 0edbfeb3c4ab49e1b0da501a288438f4: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [1105ms] delaying RPC due to: Network error: Failed to connect to peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891): Connection refused: /127.29.219.195:43891, [3555ms] refreshing cache from master, [3556ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [3560ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1835488657(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@32ac7f04 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420) -> (continuation of Deferred@271967694 after retry RPC@176644490) -> (continuation of Deferred@647911117 after retry RPC@396687525) -> (continuation of Deferred@1902804078 after retry RPC@1094435638), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@530f3802 -> wakeup thread Time-limited test -> (continuation of Deferred@226476607 after retry RPC@664413370) -> (continuation of Deferred@1887361186 after retry RPC@540003975) -> (continuation of Deferred@450785366 after retry RPC@1146871420) -> (continuation of Deferred@271967694 after retry RPC@176644490) -> (continuation of Deferred@647911117 after retry RPC@396687525) -> (continuation of Deferred@1902804078 after retry RPC@1094435638))) after lookup
23:52:25.357 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:25.358 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] Successfully connected to peer
23:52:25.392 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x8aa5abf2, L:/127.0.0.1:51852 - R:/127.29.219.194:41145] using TOKEN/null
23:52:25.439 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.195:43891
23:52:25.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:25.441331 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:43891
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33295
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:25.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.776679 32717 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:25.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.777137 32717 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:25.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.777582 32717 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:25.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.811964 32717 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:25.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.813537 32717 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.195
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:25.863415 32717 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.195:43891
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.195
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33295
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.195
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/logs
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:52:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:52:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:25.865336 32717 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:25.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:25.867796 32717 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.870114 32179 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d -> Peer 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891): Couldn't send request to peer 0edbfeb3c4ab49e1b0da501a288438f4. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.195:43891: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:52:25.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.885073 32723 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:25.886029 32724 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:27.150547 32726 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:27.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:27.153623 32725 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1263 milliseconds
23:52:27.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.153724 32717 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:52:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.155089 32717 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:27.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.157902 32717 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:27.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.159329 32717 hybrid_clock.cc:648] HybridClock initialized: now 1783468347159292 us; error 39 us; skew 500 ppm
23:52:27.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.160373 32717 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:52:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.167912 32717 webserver.cc:534] Webserver started at http://127.29.219.195:33295/ using document root <none> and password file <none>
23:52:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.168974 32717 fs_manager.cc:362] Metadata directory not provided
23:52:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.169246 32717 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:27.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.178007 32717 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
23:52:27.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.183364 32736 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.184715 32717 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
23:52:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.185142 32717 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:52:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:42 on dist-test-slave-hdrm"
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.187315 32717 fs_report.cc:389] FS layout report
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/wal
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/data
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.262822 32717 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:27.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.264657 32717 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:27.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.265290 32717 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:27.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.268389 32717 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.276038 32744 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.285422 32717 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.285708 32717 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.012s	user 0.002s	sys 0.001s
23:52:27.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.286089 32717 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.291669 32717 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.291980 32717 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.006s	sys 0.000s
23:52:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.292707 32744 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap starting.
23:52:27.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.396248 32744 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Log is configured to *not* fsync() on all Append() calls
23:52:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.524181 32744 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.525159 32744 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Bootstrap complete.
23:52:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.526868 32744 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent bootstrapping tablet: real 0.235s	user 0.162s	sys 0.065s
23:52:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.538208 32717 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.195:43891
23:52:27.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.538323   385 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.195:43891 every 8 connection(s)
23:52:27.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.541592 32717 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-2/data/info.pb
23:52:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.545089 32744 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.548496 32744 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Initialized, Role: FOLLOWER
23:52:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.549466 32744 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:27.553 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.552232 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 32717
23:52:27.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.570009 32744 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4: Time spent starting tablet: real 0.043s	user 0.020s	sys 0.011s
23:52:27.582 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:27.583 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.194:41145
23:52:27.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.584054 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 32156
23:52:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.597133   387 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:52:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.597730   387 heartbeater.cc:461] Registering TS with master...
23:52:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.599192   387 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:52:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.604928 30696 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:27.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.607574   386 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:52:27.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.607964   386 heartbeater.cc:461] Registering TS with master...
23:52:27.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.609000   386 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:52:27.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.612838 30774 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.617585   388 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:52:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.617980   388 heartbeater.cc:461] Registering TS with master...
23:52:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.619016   388 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:52:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.622428 30619 ts_manager.cc:194] Re-registered known tserver with Master: 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.625613 30619 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.195:44181
23:52:27.630 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] handling channelInactive
23:52:27.630 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state READY due to: connection closed
23:52:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:27.629410   388 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:52:27.631 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:27.631 [DEBUG - Time-limited test] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:27.636 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.636 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.637 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.655 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:27.655 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:27.656 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [6ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [6ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [24ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1837055001(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@72c260f3 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c4b19d -> wakeup thread Time-limited test)) after lookup
23:52:27.656 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:27.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.657 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.674 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:27.674 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:27.674 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [6ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [6ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [24ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [25ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [26ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [26ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [43ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1837055001(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@72c260f3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c4b19d -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405))) after lookup
23:52:27.675 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:27.676 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.676 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.676 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.696 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:27.696 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:27.696 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=65), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [6ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [6ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [24ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [25ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [26ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [26ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [43ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [44ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [45ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [45ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [64ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1837055001(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@72c260f3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c4b19d -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611))) after lookup
23:52:27.697 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:27.698 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.698 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.698 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.755 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:27.755 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:27.756 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=124), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [6ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [6ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [24ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [25ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [26ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [26ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [43ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [44ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [45ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [45ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [64ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [65ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [67ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [67ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [124ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1837055001(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@72c260f3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611) -> (continuation of Deferred@759570795 after retry RPC@263167027), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c4b19d -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611) -> (continuation of Deferred@759570795 after retry RPC@263167027))) after lookup
23:52:27.756 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:27.757 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.757 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:27.757 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:28.234 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:28.235 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)[L],d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:28.235 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=603), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [6ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [6ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [24ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [25ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [26ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [26ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [43ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [44ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [45ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [45ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [64ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [65ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [67ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [67ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [124ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [125ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [126ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [126ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [599ms] refreshing cache from master, [600ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [603ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1837055001(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@72c260f3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611) -> (continuation of Deferred@759570795 after retry RPC@263167027) -> (continuation of Deferred@952496614 after retry RPC@1274592457), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c4b19d -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611) -> (continuation of Deferred@759570795 after retry RPC@263167027) -> (continuation of Deferred@952496614 after retry RPC@1274592457))) after lookup
23:52:28.236 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] connecting to peer
23:52:28.237 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:28.237 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] cleaning up while in state CONNECTING due to: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:28.237 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145
23:52:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.674748   399 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 451a779f984c42ac861a7b36231ac58d)
23:52:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.675223   399 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.677551   399 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:28.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:28.681895 32554 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:28.686411 32554 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.690600   339 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" is_pre_election: true
23:52:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.691257   339 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 8.
23:52:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.692576 32551 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1; no voters: 451a779f984c42ac861a7b36231ac58d
23:52:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.693239   399 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 9 FOLLOWER]: Leader pre-election won for term 10
23:52:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.693517   399 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 451a779f984c42ac861a7b36231ac58d)
23:52:28.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.693828   399 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 9 FOLLOWER]: Advancing to term 10
23:52:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.698355   399 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.699940   399 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 10 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195:43891)
23:52:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.700984   339 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "d2e1fdaeeab841af80053f63388320b1" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "0edbfeb3c4ab49e1b0da501a288438f4"
23:52:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.701431   339 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 8 FOLLOWER]: Advancing to term 10
23:52:28.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:28.703173 32554 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111)
23:52:28.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.707542   339 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2e1fdaeeab841af80053f63388320b1 in term 10.
23:52:28.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.708420 32551 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, d2e1fdaeeab841af80053f63388320b1; no voters: 451a779f984c42ac861a7b36231ac58d
23:52:28.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.708962   399 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 10 FOLLOWER]: Leader election won for term 10
23:52:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.710443   399 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [term 10 LEADER]: Becoming Leader. State: Replica: d2e1fdaeeab841af80053f63388320b1, State: Running, Role: LEADER
23:52:28.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.711323   399 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:28.719434 30619 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 reported cstate change: term changed from 9 to 10, leader changed from 451a779f984c42ac861a7b36231ac58d (127.29.219.194) to d2e1fdaeeab841af80053f63388320b1 (127.29.219.193). New cstate: current_term: 10 leader_uuid: "d2e1fdaeeab841af80053f63388320b1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: HEALTHY } } }
23:52:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.109534 32554 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 -> Peer 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145): Couldn't send request to peer 451a779f984c42ac861a7b36231ac58d. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.194:41145: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:29.255 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:29.255 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:29.256 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1624), Traces: [0ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [6ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [6ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [24ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [25ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [26ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [26ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [39ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [43ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [44ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [45ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [45ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [60ms] refreshing cache from master, [60ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [64ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [65ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [67ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [67ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [124ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [125ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [126ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [126ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [599ms] refreshing cache from master, [600ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [603ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [604ms] sending RPC to server 451a779f984c42ac861a7b36231ac58d, [606ms] received response from server 451a779f984c42ac861a7b36231ac58d: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [606ms] delaying RPC due to: Network error: Failed to connect to peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145): Connection refused: /127.29.219.194:41145, [1619ms] refreshing cache from master, [1620ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [1623ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1837055001(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@72c260f3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611) -> (continuation of Deferred@759570795 after retry RPC@263167027) -> (continuation of Deferred@952496614 after retry RPC@1274592457) -> (continuation of Deferred@80168172 after retry RPC@173522505), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1c4b19d -> wakeup thread Time-limited test -> (continuation of Deferred@508632571 after retry RPC@977084405) -> (continuation of Deferred@585957007 after retry RPC@965966611) -> (continuation of Deferred@759570795 after retry RPC@263167027) -> (continuation of Deferred@952496614 after retry RPC@1274592457) -> (continuation of Deferred@80168172 after retry RPC@173522505))) after lookup
23:52:29.256 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:29.257 [DEBUG - kudu-nio-5] (Connection.java:757) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] Successfully connected to peer
23:52:29.281 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x6f636e00, L:/127.0.0.1:56106 - R:/127.29.219.193:45855] using TOKEN/null
23:52:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:29.292652   339 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 10 FOLLOWER]: Refusing update from remote peer d2e1fdaeeab841af80053f63388320b1: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
23:52:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:29.304243   419 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.001s
23:52:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:29.334945   422 mvcc.cc:204] Tried to move back new op lower bound from 7305086358687199232 to 7305086356329799680. Current Snapshot: MvccSnapshot[applied={T|T < 7305086358687199232 or (T in {7305086358687199232})}]
23:52:29.371 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.194:41145
23:52:29.371 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:29.373039 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:41145
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36759
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.705786   429 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.706247   429 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.706701   429 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.740959   429 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:29.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.741804   429 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.194
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:29.779094   429 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.194:41145
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:52:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.194
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36759
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.194
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/logs
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:52:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:29.780625   429 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:29.783041   429 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:29.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.799217   437 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:31.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:29.800973   438 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:31.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:31.090610   440 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:31.093725   439 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1288 milliseconds
23:52:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.093796   429 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:52:31.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.095078   429 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.097867   429 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.099399   429 hybrid_clock.cc:648] HybridClock initialized: now 1783468351099322 us; error 71 us; skew 500 ppm
23:52:31.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.100519   429 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:52:31.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.107239   429 webserver.cc:534] Webserver started at http://127.29.219.194:36759/ using document root <none> and password file <none>
23:52:31.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.108215   429 fs_manager.cc:362] Metadata directory not provided
23:52:31.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.108414   429 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:31.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.116010   429 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.001s
23:52:31.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.120577   448 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.121690   429 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
23:52:31.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.121971   429 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:31.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451a779f984c42ac861a7b36231ac58d"
23:52:31.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:40 on dist-test-slave-hdrm"
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.123885   429 fs_report.cc:389] FS layout report
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/wal
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/data
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.178702   429 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:31.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.180261   429 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:31.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.180686   429 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:31.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.183473   429 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.189316   455 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.197445   429 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.197686   429 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
23:52:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.197903   429 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.203184   429 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.203378   429 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
23:52:31.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.203866   455 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap starting.
23:52:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.274631   455 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Log is configured to *not* fsync() on all Append() calls
23:52:31.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.404032   455 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:52:31.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.405090   455 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Bootstrap complete.
23:52:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.407223   455 ts_tablet_manager.cc:1403] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent bootstrapping tablet: real 0.204s	user 0.140s	sys 0.055s
23:52:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.410166   429 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.194:41145
23:52:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.410272   563 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.194:41145 every 8 connection(s)
23:52:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.413789   429 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-1/data/info.pb
23:52:31.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.417037 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 429
23:52:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.448205   455 raft_consensus.cc:359] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.452174   455 raft_consensus.cc:740] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 451a779f984c42ac861a7b36231ac58d, State: Initialized, Role: FOLLOWER
23:52:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.453253   455 consensus_queue.cc:260] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.463138   455 ts_tablet_manager.cc:1434] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d: Time spent starting tablet: real 0.056s	user 0.040s	sys 0.004s
23:52:31.472 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:31.472 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.29.219.193:45855
23:52:31.473 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.473609 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 32524
23:52:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.489826   565 heartbeater.cc:344] Connected to a master server at 127.29.219.253:42131
23:52:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.490365   565 heartbeater.cc:461] Registering TS with master...
23:52:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.490767   566 heartbeater.cc:344] Connected to a master server at 127.29.219.254:45193
23:52:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.491108   566 heartbeater.cc:461] Registering TS with master...
23:52:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.491858   565 heartbeater.cc:507] Master 127.29.219.253:42131 requested a full tablet report, sending...
23:52:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.492338   566 heartbeater.cc:507] Master 127.29.219.254:45193 requested a full tablet report, sending...
23:52:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.496376 30620 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:52:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.497241 30696 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:52:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.499610 30620 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.219.194:35639
23:52:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.504624   566 heartbeater.cc:499] Master 127.29.219.254:45193 was elected leader, sending a full tablet report...
23:52:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.506615   564 heartbeater.cc:344] Connected to a master server at 127.29.219.252:38747
23:52:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.507006   564 heartbeater.cc:461] Registering TS with master...
23:52:31.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.507634   564 heartbeater.cc:507] Master 127.29.219.252:38747 requested a full tablet report, sending...
23:52:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:31.511512 30774 ts_manager.cc:194] Re-registered known tserver with Master: 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145)
23:52:31.517 [DEBUG - kudu-nio-5] (Connection.java:239) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] handling channelInactive
23:52:31.517 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state READY due to: connection closed
23:52:31.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:31.518 [DEBUG - Time-limited test] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:31.519 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.519 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.520 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.535 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:31.536 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:31.536 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1494801166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6e0a8392 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f9151c3 -> wakeup thread Time-limited test)) after lookup
23:52:31.536 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:31.537 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.537 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.538 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.555 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:31.555 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:31.555 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [20ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [20ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1494801166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6e0a8392 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f9151c3 -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684))) after lookup
23:52:31.556 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:31.557 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.557 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.557 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.575 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:31.575 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:31.575 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [20ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [20ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [39ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [39ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [57ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1494801166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6e0a8392 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f9151c3 -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296))) after lookup
23:52:31.576 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:31.577 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.577 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.577 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.695 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:31.696 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:31.696 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=178), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [20ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [20ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [39ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [39ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [57ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [58ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [59ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [59ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [173ms] refreshing cache from master, [173ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [177ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1494801166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6e0a8392 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296) -> (continuation of Deferred@176004209 after retry RPC@2126975354), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f9151c3 -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296) -> (continuation of Deferred@176004209 after retry RPC@2126975354))) after lookup
23:52:31.697 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:31.698 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.698 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:31.698 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:32.215 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:32.216 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891),451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)[L]]}]
23:52:32.216 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=698), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [20ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [20ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [39ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [39ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [57ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [58ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [59ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [59ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [173ms] refreshing cache from master, [173ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [177ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [178ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [180ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [180ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [693ms] refreshing cache from master, [693ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [697ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1494801166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6e0a8392 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296) -> (continuation of Deferred@176004209 after retry RPC@2126975354) -> (continuation of Deferred@926819043 after retry RPC@1899276735), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f9151c3 -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296) -> (continuation of Deferred@176004209 after retry RPC@2126975354) -> (continuation of Deferred@926819043 after retry RPC@1899276735))) after lookup
23:52:32.216 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] connecting to peer
23:52:32.217 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:32.217 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] cleaning up while in state CONNECTING due to: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:32.218 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855) for tablet a9a7b9a0d6cf45c78c344887dd124464: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855
23:52:32.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.563607   576 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:52:32.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.564002   576 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:32.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.566128   576 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:32.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:32.570284 32737 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:32.573750 32737 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:32.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.577945   518 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d" is_pre_election: true
23:52:32.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.578586   518 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 9.
23:52:32.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.579757 32740 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:52:32.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.580528   576 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 10 FOLLOWER]: Leader pre-election won for term 11
23:52:32.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.580828   576 raft_consensus.cc:493] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 10 FOLLOWER]: Starting leader election (detected failure of leader d2e1fdaeeab841af80053f63388320b1)
23:52:32.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.581089   576 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 10 FOLLOWER]: Advancing to term 11
23:52:32.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.587644   576 raft_consensus.cc:515] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:32.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.589411   576 leader_election.cc:290] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 11 election: Requested vote from peers 451a779f984c42ac861a7b36231ac58d (127.29.219.194:41145), d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855)
23:52:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.590070   518 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a9a7b9a0d6cf45c78c344887dd124464" candidate_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "451a779f984c42ac861a7b36231ac58d"
23:52:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.590503   518 raft_consensus.cc:3060] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 9 FOLLOWER]: Advancing to term 11
23:52:32.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:32.592790 32737 leader_election.cc:336] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111)
23:52:32.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.597462   518 raft_consensus.cc:2468] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0edbfeb3c4ab49e1b0da501a288438f4 in term 11.
23:52:32.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.598330 32740 leader_election.cc:304] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0edbfeb3c4ab49e1b0da501a288438f4, 451a779f984c42ac861a7b36231ac58d; no voters: d2e1fdaeeab841af80053f63388320b1
23:52:32.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.598980   576 raft_consensus.cc:2804] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 11 FOLLOWER]: Leader election won for term 11
23:52:32.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.600600   576 raft_consensus.cc:697] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [term 11 LEADER]: Becoming Leader. State: Replica: 0edbfeb3c4ab49e1b0da501a288438f4, State: Running, Role: LEADER
23:52:32.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.601715   576 consensus_queue.cc:237] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } }
23:52:32.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:32.610543 30620 catalog_manager.cc:5696] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 reported cstate change: term changed from 10 to 11, leader changed from d2e1fdaeeab841af80053f63388320b1 (127.29.219.193) to 0edbfeb3c4ab49e1b0da501a288438f4 (127.29.219.195). New cstate: current_term: 11 leader_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0edbfeb3c4ab49e1b0da501a288438f4" member_type: VOTER last_known_addr { host: "127.29.219.195" port: 43891 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d2e1fdaeeab841af80053f63388320b1" member_type: VOTER last_known_addr { host: "127.29.219.193" port: 45855 } health_report { overall_health: UNKNOWN } } }
23:52:33.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:33.040637 32737 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
23:52:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:33.088289   518 raft_consensus.cc:1275] T a9a7b9a0d6cf45c78c344887dd124464 P 451a779f984c42ac861a7b36231ac58d [term 11 FOLLOWER]: Refusing update from remote peer 0edbfeb3c4ab49e1b0da501a288438f4: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
23:52:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:33.089784   576 consensus_queue.cc:1048] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "451a779f984c42ac861a7b36231ac58d" member_type: VOTER last_known_addr { host: "127.29.219.194" port: 41145 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
23:52:33.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:33.103696   595 mvcc.cc:204] Tried to move back new op lower bound from 7305086358687199232 to 7305086356329799680. Current Snapshot: MvccSnapshot[applied={T|T < 7305086358687199232}]
23:52:34.275 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet a9a7b9a0d6cf45c78c344887dd124464 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498' with partition [<start>, <end>)
23:52:34.276 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a9a7b9a0d6cf45c78c344887dd124464@[0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)[L],451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145),d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)]}]
23:52:34.276 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a9a7b9a0d6cf45c78c344887dd124464, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=2758), Traces: [0ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [1ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [17ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [18ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [20ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [20ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [37ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [38ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [39ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [39ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [53ms] refreshing cache from master, [53ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [57ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [58ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [59ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [59ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [173ms] refreshing cache from master, [173ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [177ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [178ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [180ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [180ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [693ms] refreshing cache from master, [693ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [697ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, [698ms] sending RPC to server d2e1fdaeeab841af80053f63388320b1, [699ms] received response from server d2e1fdaeeab841af80053f63388320b1: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [700ms] delaying RPC due to: Network error: Failed to connect to peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855): Connection refused: /127.29.219.193:45855, [2753ms] refreshing cache from master, [2753ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.219.254:45193, [2757ms] Sub RPC GetTableLocations: received response from server master-127.29.219.254:45193: OK, deferred=Deferred@1494801166(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007f23cc36b330@6e0a8392 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296) -> (continuation of Deferred@176004209 after retry RPC@2126975354) -> (continuation of Deferred@926819043 after retry RPC@1899276735) -> (continuation of Deferred@818801713 after retry RPC@1684062918), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@6f9151c3 -> wakeup thread Time-limited test -> (continuation of Deferred@501882275 after retry RPC@2022721684) -> (continuation of Deferred@1367617836 after retry RPC@732563296) -> (continuation of Deferred@176004209 after retry RPC@2126975354) -> (continuation of Deferred@926819043 after retry RPC@1899276735) -> (continuation of Deferred@818801713 after retry RPC@1684062918))) after lookup
23:52:34.277 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] connecting to peer
23:52:34.278 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] Successfully connected to peer
23:52:34.305 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0xb979aef3, L:/127.0.0.1:50404 - R:/127.29.219.195:43891] using TOKEN/null
23:52:34.349 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.29.219.193:45855
23:52:34.349 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:34.350459 30575 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:34.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:52:34.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.657388   616 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
23:52:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.657966   616 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:52:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.658660   616 flags.cc:432] Enabled unsafe flag: --never_fsync=true
23:52:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.693375   616 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
23:52:34.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.694352   616 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.219.193
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:34.730964   616 tablet_server_runner.cc:78] Tablet server non-default flags:
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.219.212:38345
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.219.193:45855
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:52:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.219.193
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43571
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.219.254:45193,127.29.219.253:42131,127.29.219.252:38747
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.219.193
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/logs
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66915500c9e327897c442aea7b3936de4ff5c319
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Jul 2026 23:36:41 UTC on e7f111948823
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16836
23:52:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:52:34.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:34.732731   616 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:52:34.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:34.734344   616 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:52:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.746114   622 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:35.796713 32737 consensus_peers.cc:597] T a9a7b9a0d6cf45c78c344887dd124464 P 0edbfeb3c4ab49e1b0da501a288438f4 -> Peer d2e1fdaeeab841af80053f63388320b1 (127.29.219.193:45855): Couldn't send request to peer d2e1fdaeeab841af80053f63388320b1. Status: Network error: Client connection negotiation failed: client connection to 127.29.219.193:45855: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
23:52:36.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.749150   623 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:36.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:34.752017   625 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:52:36.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:36.063196   624 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1312 milliseconds
23:52:36.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.063323   616 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:52:36.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.064694   616 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:52:36.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.066995   616 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:52:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.068370   616 hybrid_clock.cc:648] HybridClock initialized: now 1783468356068332 us; error 43 us; skew 500 ppm
23:52:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.069221   616 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:52:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.075927   616 webserver.cc:534] Webserver started at http://127.29.219.193:43571/ using document root <none> and password file <none>
23:52:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.076844   616 fs_manager.cc:362] Metadata directory not provided
23:52:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.077090   616 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:52:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.085438   616 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.001s	sys 0.004s
23:52:36.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.089957   637 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
23:52:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.090962   616 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
23:52:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.091370   616 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data,/tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2e1fdaeeab841af80053f63388320b1"
23:52:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-07 23:51:38 on dist-test-slave-hdrm"
23:52:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.093348   616 fs_report.cc:389] FS layout report
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/wal
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/data
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:52:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:52:36.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.147078   616 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
23:52:36.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.148653   616 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:52:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.149194   616 kserver.cc:163] Server-wide thread pool size limit: 3276
23:52:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.151827   616 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
23:52:36.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.157688   644 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
23:52:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.165999   616 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
23:52:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.166250   616 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
23:52:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.166600   616 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
23:52:36.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.171392   616 ts_tablet_manager.cc:616] Registered 1 tablets
23:52:36.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.171648   616 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.001s	sys 0.001s
23:52:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.172101   644 tablet_bootstrap.cc:492] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Bootstrap starting.
23:52:36.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.243855   644 log.cc:826] T a9a7b9a0d6cf45c78c344887dd124464 P d2e1fdaeeab841af80053f63388320b1: Log is configured to *not* fsync() on all Append() calls
23:52:36.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.350636   616 rpc_server.cc:307] RPC server started. Bound to: 127.29.219.193:45855
23:52:36.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.350867   754 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.219.193:45855 every 8 connection(s)
23:52:36.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.353413   616 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskRjNV2q/test-tmp/mini-kudu-cluster9627047346767498253/ts-0/data/info.pb
23:52:36.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.358631 30575 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu as pid 616
23:52:36.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
23:52:36.396 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:36.413 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783468303498, tablet=a9a7b9a0d6cf45c78c344887dd124464, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
23:52:36.418 [DEBUG - main] (Connection.java:575) [peer 451a779f984c42ac861a7b36231ac58d(127.29.219.194:41145)] disconnecting while in state TERMINATED
23:52:36.419 [DEBUG - main] (Connection.java:575) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] disconnecting while in state READY
23:52:36.419 [DEBUG - main] (Connection.java:575) [peer master-127.29.219.252:38747(127.29.219.252:38747)] disconnecting while in state READY
23:52:36.419 [DEBUG - main] (Connection.java:575) [peer master-127.29.219.254:45193(127.29.219.254:45193)] disconnecting while in state READY
23:52:36.422 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.29.219.254:45193(127.29.219.254:45193)] handling channelInactive
23:52:36.426 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.219.252:38747(127.29.219.252:38747)] handling channelInactive
23:52:36.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.219.252:38747(127.29.219.252:38747)] cleaning up while in state READY due to: connection closed
23:52:36.426 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] handling channelInactive
23:52:36.427 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 0edbfeb3c4ab49e1b0da501a288438f4(127.29.219.195:43891)] cleaning up while in state READY due to: connection closed
23:52:36.428 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.219.254:45193(127.29.219.254:45193)] cleaning up while in state READY due to: connection closed
23:52:36.428 [DEBUG - main] (Connection.java:575) [peer master-127.29.219.253:42131(127.29.219.253:42131)] disconnecting while in state READY
23:52:36.428 [DEBUG - main] (Connection.java:575) [peer d2e1fdaeeab841af80053f63388320b1(127.29.219.193:45855)] disconnecting while in state TERMINATED
23:52:36.429 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:52:36.429 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.29.219.253:42131(127.29.219.253:42131)] handling channelInactive
23:52:36.429 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.219.253:42131(127.29.219.253:42131)] cleaning up while in state READY due to: connection closed
23:52:36.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.440234 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 616
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:36.474891   765 negotiation.cc:336] Failed RPC negotiation. Trace:
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.390907 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.29.219.193:42353 (local address 127.29.219.254:45193)
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.443067 (+ 52160us) server_negotiation.cc:207] Beginning negotiation
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.443086 (+    19us) server_negotiation.cc:400] Waiting for connection header
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.473587 (+ 30501us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.29.219.193:42353: BlockingRecv error: recv got EOF from 127.29.219.193:42353 (error 108)
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":51755,"thread_start_us":52117,"threads_started":1}
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:36.475257   761 negotiation.cc:336] Failed RPC negotiation. Trace:
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.363591 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.29.219.193:58405 (local address 127.29.219.253:42131)
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.385398 (+ 21807us) server_negotiation.cc:207] Beginning negotiation
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.385439 (+    41us) server_negotiation.cc:400] Waiting for connection header
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.385505 (+    66us) server_negotiation.cc:408] Connection header received
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.433173 (+ 47668us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.433224 (+    51us) server_negotiation.cc:462] Received NEGOTIATE request from client
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.433548 (+   324us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.433793 (+   245us) server_negotiation.cc:227] Negotiated authn=SASL
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.436394 (+  2601us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.437836 (+  1442us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.473709 (+ 35873us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.29.219.193:58405: BlockingRecv error: recv got EOF from 127.29.219.193:58405 (error 108)
23:52:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20892,"thread_start_us":21851,"threads_started":1}
23:52:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.477105 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 429
23:52:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260707 23:52:36.477535   759 negotiation.cc:336] Failed RPC negotiation. Trace:
23:52:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.360680 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.29.219.193:34933 (local address 127.29.219.252:38747)
23:52:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.381382 (+ 20702us) server_negotiation.cc:207] Beginning negotiation
23:52:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.381418 (+    36us) server_negotiation.cc:400] Waiting for connection header
23:52:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.388537 (+  7119us) server_negotiation.cc:408] Connection header received
23:52:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0707 23:52:36.476215 (+ 87678us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.29.219.193:34933: BlockingRecv error: recv got EOF from 127.29.219.193:34933 (error 108)
23:52:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":19912,"thread_start_us":21224,"threads_started":1}
23:52:36.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.514920 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 32717
23:52:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.545588 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 30587
23:52:36.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.576949 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 30658
23:52:36.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260707 23:52:36.607708 30575 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskRjNV2q/build/tsan/bin/kudu with pid 30735
23:52:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-07T23:52:37Z chronyd exiting

Time: 71.325

OK (1 test)

